# Make warnings not displayed
import warnings
warnings.filterwarnings("ignore")

# Import display and MarkDown from IPython to display the messages in text cell via python code block
from IPython.display import display, Markdown, Image

# Import numpy for numerical python operation
import numpy as np

# Import pandas for data loading and dataframe activities
import pandas as pd

# Import seaborn and pyplot from matplotlib to visualize the data
import seaborn as sns
from matplotlib import pyplot as plt
%matplotlib inline

from sklearn.preprocessing import StandardScaler

# Import train_test_split from sklearn.model_selection for splitting the data into train and test split along with KFold, cross_val_score, GridSearchCV, RandomizedSearchCV
from sklearn.model_selection import train_test_split, KFold, StratifiedKFold, cross_validate, cross_val_score, GridSearchCV, RandomizedSearchCV

# Library to import Pipeline
from imblearn.pipeline import Pipeline

# Library to import different metrics to measure the metrics
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, make_scorer, confusion_matrix, roc_curve, auc

# Library to import DecisionTreeClassifier to build the DecisionTreeClassifier
from sklearn.tree import DecisionTreeClassifier

# Import Library to calculate BaggingClassifier, AdaBoostClassifier, GradientClassifier
from sklearn.ensemble import BaggingClassifier, AdaBoostClassifier, GradientBoostingClassifier

# Library for XGBoost ensemble technique
from xgboost import XGBClassifier

# Import SMOTE from imblearn.over_sampling
from imblearn.over_sampling import SMOTE

# Import RandomUnderSampler from imblearn.under_sampling
from imblearn.under_sampling import RandomUnderSampler

# Import sp_randint from scipy.stats
from scipy.stats import randint

# Define UnivariateAnalyzerContinuous to perofrm the univariate analysis on a continuous column
def UnivariateAnalyzerContinuous(ColumnName, Category):
  fig, axes = plt.subplots(1, 4, figsize = (20, 5))
  sns.histplot(data = BankChurnABT, x = ColumnName, kde = True, ax = axes[0])
  axes[0].set_title(ColumnName)
  plt.setp(axes[0].xaxis.get_majorticklabels(), rotation=45)
  sns.histplot(data = BankChurnABT, x = ColumnName, kde = True, hue = Category, ax = axes[1])
  axes[1].set_title(ColumnName)
  plt.setp(axes[1].xaxis.get_majorticklabels(), rotation=45)
  sns.boxplot(data = BankChurnABT, x = ColumnName, ax = axes[2])
  axes[2].set_title(ColumnName)
  plt.setp(axes[2].xaxis.get_majorticklabels(), rotation=45)
  sns.boxplot(data = BankChurnABT, x = ColumnName, hue = Category, ax = axes[3])
  axes[3].set_title(ColumnName)
  plt.setp(axes[3].xaxis.get_majorticklabels(), rotation=45)
  plt.show()

# Define UnivariateAnalyzerCategorical to perofrm the univariate analysis on a categorical column
def UnivariateAnalyzerCategorical(ColumnName, Category):
  fig, axes = plt.subplots(1, 2, figsize = (10, 5))
  sns.countplot(data = BankChurnABT, x = ColumnName, ax = axes[0])
  plt.setp(axes[0].xaxis.get_majorticklabels(), rotation=45)
  axes[0].set_title(ColumnName)
  sns.countplot(data = BankChurnABT, x = ColumnName, hue = Category, ax = axes[1])
  plt.setp(axes[1].xaxis.get_majorticklabels(), rotation=45)
  axes[1].set_title(ColumnName)
  plt.show()

# defining a function to compute different metrics to check performance of a classification model built using sklearn
def ModelPerformanceMetrics(model, predictors, target):
  # predicting using the independent variables
  pred = model.predict(predictors)

  accuracy = accuracy_score(target, pred)  # to compute Accuracy
  recall = recall_score(target, pred)  # to compute Recall
  precision = precision_score(target, pred)  # to compute Precision
  f1 = f1_score(target, pred)  # to compute F1-score

  # creating a dataframe of metrics
  df_perf = pd.DataFrame({"Accuracy": accuracy, "Recall": recall, "Precision": precision, "F1": f1,}, index=[0],)

  return df_perf

def ConfusionMatrix(model, predictors, target):
  y_pred = model.predict(predictors)
  cm = confusion_matrix(target, y_pred)
  labels = np.asarray([["{0:0.0f}".format(item) + "\n{0:.2%}".format(item / cm.flatten().sum())] for item in cm.flatten()]).reshape(2, 2)
  # plot the matrix inside a heatmap
  plt.figure(figsize=(6, 4))
  sns.heatmap(cm, annot=labels, fmt="")
  plt.ylabel("True label")
  plt.xlabel("Predicted label")
  plt.show()

# Read the file BankChurners.csv from google drive and load into a dataframe
FileURL = "/content/drive/MyDrive/Colab Notebooks/BankChurners.csv"
BankChurnersRaw = pd.read_csv(FileURL)
if BankChurnersRaw.empty:
  display(Markdown("**Observations:**\n- Errors occurred while loading the data and the data loading is unsuccessful."))
else:
  display(Markdown("**Observations:**\n- No errors occurred while loading the data and the data loading is successful."))

# Get the rows and columns in the dataframe
display(Markdown("**Observations:**\n- There are {} rows and {} columns in the dataframe.".format(BankChurnersRaw.shape[0], BankChurnersRaw.shape[1])))

#Prepare the ABT (Analysis Base Table by copying the dataframe)
BankChurnABT = BankChurnersRaw.copy()
display(Markdown("**Observations:**\n- There are {} rows and {} columns in the ABT (Analysis Base Table).".format(BankChurnABT.shape[0], BankChurnABT.shape[1])))

# Display the column names and datatypes of of the ABT
display(Markdown("**Observations:**\n"))
for c in range(0, BankChurnABT.dtypes.shape[0]):
  display(Markdown("- **Column Name:**{} **Data Type:** {}".format(BankChurnABT.dtypes.index[c],BankChurnABT.dtypes.iloc[c])))

#Use the describe and transpose to display the 5 point summary of the number datatypes
BankChurnABT.describe().transpose()

if BankChurnABT["CLIENTNUM"].count() == BankChurnABT["CLIENTNUM"].nunique():
  display(Markdown("**Observations**\n- This is a column that has unique values and we can drop this column."))

# Drop the column CLIENTNUM
BankChurnABT = BankChurnABT.drop("CLIENTNUM", axis = 1)
ColumnDetails = BankChurnABT.dtypes
display(Markdown("**Observations:**\n- There are {} columns in the dataframe.".format(len(ColumnDetails))))

# Print the unique values in the categorical column Attrition_Flag
print("Unique values and the counts are: ", BankChurnABT["Attrition_Flag"].value_counts())
# Print the total null values
print("The count of null values are:", BankChurnABT["Attrition_Flag"].isnull().sum())

Unique values and the counts are:  Attrition_Flag
Existing Customer    8500
Attrited Customer    1627
Name: count, dtype: int64
The count of null values are: 0

# Plot the countplot for the continuous column Attrition_Flag
sns.countplot(data = BankChurnABT, x = "Attrition_Flag")
plt.title("Attrition_Flag")
plt.show()

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Customer_Age"].isnull().sum())
# Five-Point summary for the column Customer_age
BankChurnABT["Customer_Age"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the continuous column Customer_Age
UnivariateAnalyzerContinuous("Customer_Age", "Attrition_Flag")

# Since the values are limited we can consider this as a categorical attribute as  well
plt.figure(figsize = (50, 5))
sns.countplot(data = BankChurnABT, x = "Customer_Age", hue = "Attrition_Flag")
plt.show()

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Gender"].isnull().sum())
# Print the unique values and the counts in the categorical column Gender
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Gender"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 0
Unique values and counts are: 
 Attrition_Flag  Attrited Customer  Existing Customer    All
Gender                                                     
F                             930               4428   5358
M                             697               4072   4769
All                          1627               8500  10127

# Plot the countplot for the categorical column Gender
UnivariateAnalyzerCategorical("Gender", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Dependent_count"].isnull().sum())
# Five-Point summary for the column Dependent_count
BankChurnABT["Dependent_count"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Dependent_count
UnivariateAnalyzerContinuous("Dependent_count", "Attrition_Flag")

# Since the values are limited we can consider this as a categorical attribute as  well
UnivariateAnalyzerCategorical("Dependent_count", "Attrition_Flag")
# Print the unique values and the counts in the categorical column Gender
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Dependent_count"], BankChurnABT["Attrition_Flag"], margins=True))

Unique values and counts are: 
 Attrition_Flag   Attrited Customer  Existing Customer    All
Dependent_count                                             
0                              135                769    904
1                              269               1569   1838
2                              417               2238   2655
3                              482               2250   2732
4                              260               1314   1574
5                               64                360    424
All                           1627               8500  10127

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Education_Level"].isnull().sum())
# Print the unique values and the counts in the categorical column Education_Level
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Education_Level"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 1519
Unique values and counts are: 
 Attrition_Flag   Attrited Customer  Existing Customer   All
Education_Level                                            
College                        154                859  1013
Doctorate                       95                356   451
Graduate                       487               2641  3128
High School                    306               1707  2013
Post-Graduate                   92                424   516
Uneducated                     237               1250  1487
All                           1371               7237  8608

# Plot the countplot for the categorical column Education_Level
UnivariateAnalyzerCategorical("Education_Level", "Attrition_Flag")

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Marital_Status"].isnull().sum())
# Print the unique values and the counts in the categorical column Marital_Status
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Marital_Status"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 749
Unique values and counts are: 
 Attrition_Flag  Attrited Customer  Existing Customer   All
Marital_Status                                            
Divorced                      121                627   748
Married                       709               3978  4687
Single                        668               3275  3943
All                          1498               7880  9378

# Plot the countplot for the categorical column Marital_Status
UnivariateAnalyzerCategorical("Marital_Status", "Attrition_Flag")

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Income_Category"].isnull().sum())
# Print the unique values and the counts in the categorical column Income_Category
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Income_Category"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 0
Unique values and counts are: 
 Attrition_Flag   Attrited Customer  Existing Customer    All
Income_Category                                             
$120K +                        126                601    727
$40K - $60K                    271               1519   1790
$60K - $80K                    189               1213   1402
$80K - $120K                   242               1293   1535
Less than $40K                 612               2949   3561
abc                            187                925   1112
All                           1627               8500  10127

# Plot the countplot for the categorical column Income_Category
UnivariateAnalyzerCategorical("Income_Category", "Attrition_Flag")

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Card_Category"].isnull().sum())
# Print the unique values and the counts in the categorical column Card_Category
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Card_Category"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 0
Unique values and counts are: 
 Attrition_Flag  Attrited Customer  Existing Customer    All
Card_Category                                              
Blue                         1519               7917   9436
Gold                           21                 95    116
Platinum                        5                 15     20
Silver                         82                473    555
All                          1627               8500  10127

# Plot the countplot for the categorical column Card_Category
UnivariateAnalyzerCategorical("Card_Category", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Months_on_book"].isnull().sum())
# Five-Point summary for the column Months_on_book
BankChurnABT["Months_on_book"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Months_on_book
UnivariateAnalyzerContinuous("Months_on_book", "Attrition_Flag")

# Since the values are limited we can consider this as a categorical attribute as  well
plt.figure(figsize = (50, 10))
sns.countplot(data = BankChurnABT, x = "Months_on_book", hue = "Attrition_Flag")
plt.show()

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Total_Relationship_Count"].isnull().sum())
# Five-Point summary for the column Total_Relationship_Count
BankChurnABT["Total_Relationship_Count"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Total_Relationship_Count
UnivariateAnalyzerContinuous("Total_Relationship_Count", "Attrition_Flag")

# Print the unique values and the counts in the categorical column Card_Category
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Total_Relationship_Count"], BankChurnABT["Attrition_Flag"], margins=True))
# Plot the histplot and boxplot for the column Total_Relationship_Count
UnivariateAnalyzerCategorical("Total_Relationship_Count", "Attrition_Flag")

Unique values and counts are: 
 Attrition_Flag            Attrited Customer  Existing Customer    All
Total_Relationship_Count                                             
1                                       233                677    910
2                                       346                897   1243
3                                       400               1905   2305
4                                       225               1687   1912
5                                       227               1664   1891
6                                       196               1670   1866
All                                    1627               8500  10127

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Months_Inactive_12_mon"].isnull().sum())
# Five-Point summary for the column Total_Relationship_Count
BankChurnABT["Months_Inactive_12_mon"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Months_Inactive_12_mon
UnivariateAnalyzerContinuous("Months_Inactive_12_mon", "Attrition_Flag")

# Since this attribute has a few distinct numbers we can consider this as a cateogrical column
UnivariateAnalyzerCategorical("Months_Inactive_12_mon", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Contacts_Count_12_mon"].isnull().sum())
# Five-Point summary for the column Contacts_Count_12_mon
BankChurnABT["Contacts_Count_12_mon"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Contacts_Count_12_mon
UnivariateAnalyzerContinuous("Contacts_Count_12_mon", "Attrition_Flag")

# This attribute has a few unique values and we can consider this as a categorical attribute
UnivariateAnalyzerCategorical("Contacts_Count_12_mon", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Credit_Limit"].isnull().sum())
# Five-Point summary for the column Credit_Limit
BankChurnABT["Credit_Limit"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Credit_Limit
UnivariateAnalyzerContinuous("Credit_Limit", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Total_Revolving_Bal"].isnull().sum())
# Five-Point summary for the column Total_Revolving_Bal
BankChurnABT["Total_Revolving_Bal"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Total_Revolving_Bal
UnivariateAnalyzerContinuous("Total_Revolving_Bal", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Avg_Open_To_Buy"].isnull().sum())
# Five-Point summary for the column Avg_Open_To_Buy
BankChurnABT["Avg_Open_To_Buy"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Avg_Open_To_Buy
UnivariateAnalyzerContinuous("Avg_Open_To_Buy", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Total_Trans_Amt"].isnull().sum())
# Five-Point summary for the column Total_Trans_Amt
BankChurnABT["Total_Trans_Amt"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Total_Trans_Amt
UnivariateAnalyzerContinuous("Total_Trans_Amt", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Total_Trans_Ct"].isnull().sum())
# Five-Point summary for the column Total_Trans_Ct
BankChurnABT["Total_Trans_Ct"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Total_Trans_Ct
UnivariateAnalyzerContinuous("Total_Trans_Ct", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Total_Ct_Chng_Q4_Q1"].isnull().sum())
# Five-Point summary for the column Total_Ct_Chng_Q4_Q1
BankChurnABT["Total_Ct_Chng_Q4_Q1"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Total_Ct_Chng_Q4_Q1
UnivariateAnalyzerContinuous("Total_Ct_Chng_Q4_Q1", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Total_Amt_Chng_Q4_Q1"].isnull().sum())
# Five-Point summary for the column Total_Amt_Chng_Q4_Q1
BankChurnABT["Total_Amt_Chng_Q4_Q1"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Total_Amt_Chng_Q4_Q1
UnivariateAnalyzerContinuous("Total_Amt_Chng_Q4_Q1", "Attrition_Flag")

# This is a continuous column and we can print the null values
print("The count of null values are:", BankChurnABT["Avg_Utilization_Ratio"].isnull().sum())
# Five-Point summary for the column Avg_Utilization_Ratio
BankChurnABT["Avg_Utilization_Ratio"].describe().transpose()

The count of null values are: 0

# Plot the histplot and boxplot for the column Avg_Utilization_Ratio
UnivariateAnalyzerContinuous("Avg_Utilization_Ratio", "Attrition_Flag")

#Draw a pairplot between all the numeric columns
sns.pairplot(data = BankChurnABT, hue = "Attrition_Flag")
plt.show()

# Show the correltion between the numeric columns
plt.figure(figsize=(20, 20))
sns.heatmap(data = BankChurnABT[["Customer_Age", "Dependent_count", "Months_on_book", "Total_Relationship_Count", "Months_Inactive_12_mon", "Contacts_Count_12_mon", "Credit_Limit", "Total_Revolving_Bal", "Avg_Open_To_Buy", "Total_Trans_Amt", "Total_Trans_Ct", "Total_Ct_Chng_Q4_Q1", "Total_Amt_Chng_Q4_Q1", "Avg_Utilization_Ratio"]].corr(), annot = True)
plt.show()

# Group the correlations into strong and weak based on the table obtained above
Corr = BankChurnABT[["Customer_Age", "Dependent_count", "Months_on_book", "Total_Relationship_Count", "Months_Inactive_12_mon", "Contacts_Count_12_mon", "Credit_Limit", "Total_Revolving_Bal", "Avg_Open_To_Buy", "Total_Trans_Amt", "Total_Trans_Ct", "Total_Ct_Chng_Q4_Q1", "Total_Amt_Chng_Q4_Q1", "Avg_Utilization_Ratio"]].corr()
CorrSize = len(Corr)
StrongPositiveCorr = ""
ModeratePositiveCorr = ""
WeakPositiveCorr = ""
NoCorr = ""
WeakNegativeCorr = ""
ModerateNegativeCorr = ""
StrongNegativeCorr = ""
UndefinedCorr = ""
for i in range(0, CorrSize):
  for j in range(0, i):
    CorrVal = float(Corr.iloc[i, j])
    if  CorrVal <= 1 and CorrVal >= 0.75:
      StrongPositiveCorr = StrongPositiveCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    elif CorrVal < 0.75 and CorrVal >= 0.5:
      ModeratePositiveCorr = ModeratePositiveCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    elif CorrVal < 0.50 and CorrVal >= 0.25:
      WeakPositiveCorr = WeakPositiveCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    elif CorrVal < 0.25 and CorrVal > -0.25:
      NoCorr = NoCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    elif CorrVal <= -0.25 and CorrVal > -0.5:
      WeakNegativeCorr = WeakNegativeCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    elif CorrVal <= -0.5 and CorrVal > -0.75:
      ModerateNegativeCorr = ModerateNegativeCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    elif CorrVal <= -0.75 and CorrVal >= -1:
      StrongNegativeCorr = StrongNegativeCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
    else:
      UndefinedCorr = UndefinedCorr + "- Correlation between " + Corr.index[i] + " vs " + Corr.columns[j] + " : " + str(Corr.iloc[i, j]) + "\n"
if StrongPositiveCorr != "":
  display(Markdown("**Pair of columns that Show Strong Positive Correlation:**\n{}".format(StrongPositiveCorr)))
if ModeratePositiveCorr != "":
  display(Markdown("**Pair of columns that Show Moderate Positive Correlation:**\n{}".format(ModeratePositiveCorr)))
if WeakPositiveCorr != "":
  display(Markdown("**Pair of columns that Show Weak Positive Correlation:**\n{}".format(WeakPositiveCorr)))
if NoCorr != "":
  display(Markdown("**Pair of columns that Show No Correlation:**\n{}".format(NoCorr)))
if WeakNegativeCorr != "":
  display(Markdown("**Pair of columns that show Weak Negative Correlation:**\n{}".format(WeakNegativeCorr)))
if ModerateNegativeCorr != "":
  display(Markdown("**Pair of columns that Show Moderate Negative Correlation:**\n{}".format(ModerateNegativeCorr)))
if StrongNegativeCorr != "":
  display(Markdown("**Pair of columns that Show Strong Negative Correlation:**\n{}".format(StrongNegativeCorr)))
if UndefinedCorr != "":
  display(Markdown("**Pair of columns that Show Undefined Correlation:**\n{}".format(UndefinedCorr)))

# # Check the columns that have null values
BankChurnABT.isnull().sum()

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Education_Level"].isnull().sum())
# Print the unique values and the counts in the categorical column Education_Level
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Education_Level"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 1519
Unique values and counts are: 
 Attrition_Flag   Attrited Customer  Existing Customer   All
Education_Level                                            
College                        154                859  1013
Doctorate                       95                356   451
Graduate                       487               2641  3128
High School                    306               1707  2013
Post-Graduate                   92                424   516
Uneducated                     237               1250  1487
All                           1371               7237  8608

MFEducationLevel = BankChurnABT["Education_Level"].mode()[0]

# Replace NaN values in column Education_Level with the most frequent value
BankChurnABT["Education_Level"].fillna(MFEducationLevel, inplace = True)

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Education_Level"].isnull().sum())

# Print the unique values and the counts in the categorical column Education_Level
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Education_Level"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 0
Unique values and counts are: 
 Attrition_Flag   Attrited Customer  Existing Customer    All
Education_Level                                             
College                        154                859   1013
Doctorate                       95                356    451
Graduate                       743               3904   4647
High School                    306               1707   2013
Post-Graduate                   92                424    516
Uneducated                     237               1250   1487
All                           1627               8500  10127

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Marital_Status"].isnull().sum())
# Print the unique values and the counts in the categorical column Marital_Status
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Marital_Status"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 749
Unique values and counts are: 
 Attrition_Flag  Attrited Customer  Existing Customer   All
Marital_Status                                            
Divorced                      121                627   748
Married                       709               3978  4687
Single                        668               3275  3943
All                          1498               7880  9378

MFMaritalStatus = BankChurnABT["Marital_Status"].mode()[0]

# Replace NaN values in column Marital_Status with the most frequent value
BankChurnABT["Marital_Status"].fillna(MFMaritalStatus, inplace = True)

# Print the total null values in the column
print("The count of null values are:", BankChurnABT["Marital_Status"].isnull().sum())

# Print the unique values and the counts in the categorical column Marital_Status
print("Unique values and counts are: \n", pd.crosstab(BankChurnABT["Marital_Status"], BankChurnABT["Attrition_Flag"], margins=True))

The count of null values are: 0
Unique values and counts are: 
 Attrition_Flag  Attrited Customer  Existing Customer    All
Marital_Status                                             
Divorced                      121                627    748
Married                       838               4598   5436
Single                        668               3275   3943
All                          1627               8500  10127

# Identify the rows that have Income_Category as abc and replace them with the majority category
print("The column with dummy values: ", BankChurnABT[BankChurnABT["Income_Category"] == "abc"]["Income_Category"].value_counts())
BankChurnABT["Income_Category"].replace("abc", np.nan, inplace = True)
MFIncomeLevel = BankChurnABT["Income_Category"].mode()[0]

# Replace NaN values in column Income_Category with the most frequent value
BankChurnABT["Income_Category"].fillna(MFIncomeLevel, inplace = True)

# Print the total null values in the column
print("The distinct values in the column: ", BankChurnABT["Income_Category"].value_counts())

The column with dummy values:  Income_Category
abc    1112
Name: count, dtype: int64
The distinct values in the column:  Income_Category
Less than $40K    4673
$40K - $60K       1790
$80K - $120K      1535
$60K - $80K       1402
$120K +            727
Name: count, dtype: int64

# Analysis of the data having Months_on_book = 36
DuplicatedRecords = BankChurnABT[BankChurnABT["Months_on_book"] == 36].duplicated()
DuplicatedRecords.value_counts()

# Identify the columns that have non-numeric categorical value
print("The columns with non-numeric categorical value:", list(BankChurnABT.dtypes[BankChurnABT.dtypes == "object"].index))

The columns with non-numeric categorical value: ['Attrition_Flag', 'Gender', 'Education_Level', 'Marital_Status', 'Income_Category', 'Card_Category']

print("List of unique values in Attrition Flag:", list(BankChurnABT["Attrition_Flag"].unique()))
print("List of unique values in Gender:", list(BankChurnABT["Gender"].unique()))
print("List of unique values in Education_Level:", list(BankChurnABT["Education_Level"].unique()))
print("List of unique values in Marital_Status:", list(BankChurnABT["Marital_Status"].unique()))
print("List of unique values in Income_Category:", list(BankChurnABT["Income_Category"].unique()))
print("List of unique values in Card_Category:", list(BankChurnABT["Card_Category"].unique()))

List of unique values in Attrition Flag: ['Existing Customer', 'Attrited Customer']
List of unique values in Gender: ['M', 'F']
List of unique values in Education_Level: ['High School', 'Graduate', 'Uneducated', 'College', 'Post-Graduate', 'Doctorate']
List of unique values in Marital_Status: ['Married', 'Single', 'Divorced']
List of unique values in Income_Category: ['$60K - $80K', 'Less than $40K', '$80K - $120K', '$40K - $60K', '$120K +']
List of unique values in Card_Category: ['Blue', 'Gold', 'Silver', 'Platinum']

# Prepare the replaceStruct and OneHotCols to replace the above categorical values with the boolean values
replaceStruct = {"Attrition_Flag": {"Existing Customer": False, "Attrited Customer": True }}
oneHotCols = ["Gender", "Education_Level", "Marital_Status", "Income_Category", "Card_Category"]
BankChurnABT = BankChurnABT.replace(replaceStruct)
BankChurnABT = pd.get_dummies(BankChurnABT, columns = oneHotCols)
BankChurnABT.dtypes

# Split the Datafraome into X and Y with Attrition_Flag as Y and rest of the column as X
X = BankChurnABT.drop(columns = "Attrition_Flag")
Y = BankChurnABT["Attrition_Flag"]

# Split the Dataframe X and Y into XTemp, XTest, YTemp, YTest
XTemp, XTest, YTemp, YTest = train_test_split(X, Y, test_size = 0.20, random_state = 1, stratify = Y)
# Split the Dataframe XTemp and YTemp into XTrain, XValid, YTrain, YValid
XTrain, XValid, YTrain, YValid = train_test_split(XTemp, YTemp, test_size = 0.20, random_state = 1, stratify = YTemp)

# Print the rows and columns in the original dataframe and the split dataframe
print("The number of rows in the dataframe X:", X.shape[0], "and the number of columns in the dataframe X:", X.shape[1])
print("The number of rows in the dataframe Y:", Y.shape[0])

# Print the rows and columns in the XTrain, XValid, XTest, YTrain, YValid, YTest
print("The number of rows in the dataframe XTrain:", XTrain.shape[0], "and the number of columns in the dataframe XTrain:", XTrain.shape[1])
print("The number of rows in the dataframe XValid:", XValid.shape[0], "and the number of columns in the dataframe XValid:", XValid.shape[1])
print("The number of rows in the dataframe XTest:", XTest.shape[0], "and the number of columns in the dataframe XTest:", XTest.shape[1])
print("The number of rows in the dataframe YTrain:", YTrain.shape[0])
print("The number of rows in the dataframe YValid:", YValid.shape[0])
print("The number of rows in the dataframe YTest:", YTest.shape[0])

# Print the total rows from all the dataframes XTrain, XValid, XTest and YTrain, YValid, YTest
print("The total rows from all dataframe XTrain, XValid, XTest: ", XTrain.shape[0] + XValid.shape[0] + XTest.shape[0])
print("The total rows from all dataframe YTrain, YValid, YTest: ", YTrain.shape[0] + YValid.shape[0] + YTest.shape[0])

The number of rows in the dataframe X: 10127 and the number of columns in the dataframe X: 34
The number of rows in the dataframe Y: 10127
The number of rows in the dataframe XTrain: 6480 and the number of columns in the dataframe XTrain: 34
The number of rows in the dataframe XValid: 1621 and the number of columns in the dataframe XValid: 34
The number of rows in the dataframe XTest: 2026 and the number of columns in the dataframe XTest: 34
The number of rows in the dataframe YTrain: 6480
The number of rows in the dataframe YValid: 1621
The number of rows in the dataframe YTest: 2026
The total rows from all dataframe XTrain, XValid, XTest:  10127
The total rows from all dataframe YTrain, YValid, YTest:  10127

# We will use this class to store the different matrics of various models and compare
class ModelPerformanceStats:
  def __init__(self, name, accuracy, recall, precision, f1):
    self.name = name
    self.accuracy = accuracy
    self.recall = recall
    self.precision = precision
    self.f1 = f1
  def __repr__(self):
    return f"ModelPerformanceStats(name = {self.name}, accuracy = {self.accuracy}, recall = {self.recall}, precision = {self.precision}, f1 = {self.f1})"

AllModels = []

# Use Pipeline() to club scaling and model building

DefaultModels = []
DefaultModelNames = []
DefaultModelResults = []

DefaultModels.append(("DTCDefaultModel", Pipeline([("DTCScaler", StandardScaler()), ("DTCDefault", DecisionTreeClassifier(random_state = 1))])))
DefaultModels.append(("BCDefaultModel", Pipeline([("BCScaler", StandardScaler()), ("BCDefault", BaggingClassifier(random_state = 1))])))
DefaultModels.append(("ABCDefaultModel", Pipeline([("ABCScaler", StandardScaler()), ("ABCDefault", AdaBoostClassifier(random_state = 1))])))
DefaultModels.append(("GBCDefaultModel", Pipeline([("GBCScaler", StandardScaler()), ("GBCDefault", GradientBoostingClassifier(random_state = 1))])))
DefaultModels.append(("XGBCCDefaultModel", Pipeline([("XGBCScaler", StandardScaler()), ("XGBCDefault", XGBClassifier(random_state = 1))])))

# loop through all models to get the mean cross validated score
display(Markdown("**Cross-Validation Performance:**"))

for name, model in DefaultModels:
  scoring = ["accuracy", "recall", "precision", "f1"]
  kfold = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 1)
  cv_result = cross_validate(estimator = model, X = XTrain, y = YTrain, scoring = scoring, cv = kfold)
  DefaultModelResults.append(cv_result["test_recall"])
  DefaultModelNames.append(name)
  display(Markdown("Model Name: {}".format(name)))
  display(Markdown("- Recall: {} Min: {} Max: {} Mean: {}".format(cv_result["test_recall"], cv_result["test_recall"].min(), cv_result["test_recall"].max(), cv_result["test_recall"].mean())))

# loop through all models to get the
display(Markdown("**Validation Performance: with Recall Values:**"))
plt.figure(figsize=(10, 7))
for name, model in DefaultModels:
  model.fit(XTrain, YTrain)
  YPred = model.predict(XValid)
  accuracy = accuracy_score(YValid, YPred)
  recall = recall_score(YValid, YPred)
  precision = precision_score(YValid, YPred)
  f1 = f1_score(YValid, YPred)
  AllModels.append(ModelPerformanceStats(name, accuracy, recall, precision, f1))
  display(Markdown("- Model Name: {} Recall: {}".format(name, recall)))
  fpr, tpr, thresholds = roc_curve(YValid, YPred)
  roc_auc = auc(fpr, tpr)
  plt.plot(fpr, tpr, label=f"{name} (AUC = {roc_auc:.2f})")

# Plot settings
plt.plot([0, 1], [0, 1], color='navy', linestyle='--')  # Diagonal line for random guessing
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate (FPR)')
plt.ylabel('True Positive Rate (TPR)')
plt.title('Receiver Operating Characteristic (ROC) Curve')
plt.legend(loc='lower right')
plt.show()

# Plotting boxplots for CV scores of all models defined above
fig = plt.figure(figsize=(10, 7))

fig.suptitle("CV Comparison")
ax = fig.add_subplot(111)

plt.boxplot(DefaultModelResults)
ax.set_xticklabels(DefaultModelNames)

plt.show()

# Use Pipeline() to club scaling, oversampling and model building

OSDefaultModels = []
OSDefaultModelNames = []
OSDefaultModelResults = []

OSDefaultModels.append(("OSDTCDefaultModel", Pipeline([("OSDTCScaler", StandardScaler()), ("OSDTCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSDTCDefault", DecisionTreeClassifier(random_state = 1))])))
OSDefaultModels.append(("OSBCDefaultModel", Pipeline([("OSBCScaler", StandardScaler()), ("OSBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSBCDefault", BaggingClassifier(random_state = 1))])))
OSDefaultModels.append(("OSABCDefaultModel", Pipeline([("OSABCScaler", StandardScaler()), ("OSABCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSABCDefault", AdaBoostClassifier(random_state = 1))])))
OSDefaultModels.append(("OSGBCDefaultModel", Pipeline([("OSGBCScaler", StandardScaler()), ("OSGBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSGBCDefault", GradientBoostingClassifier(random_state = 1))])))
OSDefaultModels.append(("OSXGBCCDefaultModel", Pipeline([("OSXGBCScaler", StandardScaler()), ("OSXGBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSXGBCDefault", XGBClassifier(random_state = 1))])))

# loop through all models to get the mean cross validated score
display(Markdown("**Cross-Validation Performance:**"))

for name, model in OSDefaultModels:
  scoring = ["accuracy", "recall", "precision", "f1"]
  kfold = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 1)
  cv_result = cross_validate(estimator = model, X = XTrain, y = YTrain, scoring = scoring, cv = kfold)
  OSDefaultModelResults.append(cv_result["test_recall"])
  OSDefaultModelNames.append(name)
  display(Markdown("Model Name: {}".format(name)))
  display(Markdown("- Recall: {} Min: {} Max: {} Mean: {}".format(cv_result["test_recall"], cv_result["test_recall"].min(), cv_result["test_recall"].max(), cv_result["test_recall"].mean())))

display(Markdown("**Validation Performance: with Recall Values:**"))
plt.figure(figsize=(10, 7))
for name, model in OSDefaultModels:
  model.fit(XTrain, YTrain)
  YPred = model.predict(XValid)
  accuracy = accuracy_score(YValid, YPred)
  recall = recall_score(YValid, YPred)
  precision = precision_score(YValid, YPred)
  f1 = f1_score(YValid, YPred)
  AllModels.append(ModelPerformanceStats(name, accuracy, recall, precision, f1))
  display(Markdown("- Model Name: {} Recall: {}".format(name, recall)))
  fpr, tpr, thresholds = roc_curve(YValid, YPred)
  roc_auc = auc(fpr, tpr)
  plt.plot(fpr, tpr, label=f"{name} (AUC = {roc_auc:.2f})")

# Plot settings
plt.plot([0, 1], [0, 1], color='navy', linestyle='--')  # Diagonal line for random guessing
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate (FPR)')
plt.ylabel('True Positive Rate (TPR)')
plt.title('Receiver Operating Characteristic (ROC) Curve')
plt.legend(loc='lower right')
plt.show()

# Plotting boxplots for CV scores of all models defined above
fig = plt.figure(figsize=(10, 7))

fig.suptitle("CV Comparison")
ax = fig.add_subplot(111)

plt.boxplot(OSDefaultModelResults)
ax.set_xticklabels(OSDefaultModelNames)

plt.show()

# Use Pipeline() to club scaling, oversampling and model building

USDefaultModels = []
USDefaultModelNames = []
USDefaultModelResults = []

USDefaultModels.append(("USDTCDefaultModel", Pipeline([("USDTCScaler", StandardScaler()), ("USDTCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USDTCDefault", DecisionTreeClassifier(random_state = 1))])))
USDefaultModels.append(("USBCDefaultModel", Pipeline([("USBCScaler", StandardScaler()), ("USBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USBCDefault", BaggingClassifier(random_state = 1))])))
USDefaultModels.append(("USABCDefaultModel", Pipeline([("USABCScaler", StandardScaler()), ("USABCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USABCDefault", AdaBoostClassifier(random_state = 1))])))
USDefaultModels.append(("USGBCDefaultModel", Pipeline([("USGBCScaler", StandardScaler()), ("USGBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USGBCDefault", GradientBoostingClassifier(random_state = 1))])))
USDefaultModels.append(("USXGBCCDefaultModel", Pipeline([("USXGBCScaler", StandardScaler()), ("USXGBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USXGBCDefault", XGBClassifier(random_state = 1))])))

# loop through all models to get the mean cross validated score
display(Markdown("**Cross-Validation Performance:**"))

for name, model in USDefaultModels:
  scoring = ["accuracy", "recall", "precision", "f1"]
  kfold = StratifiedKFold(n_splits = 10, shuffle = True, random_state = 1)
  cv_result = cross_validate(estimator = model, X = XTrain, y = YTrain, scoring = scoring, cv = kfold)
  USDefaultModelResults.append(cv_result["test_recall"])
  USDefaultModelNames.append(name)
  display(Markdown("Model Name: {}".format(name)))
  display(Markdown("- Recall: {} Min: {} Max: {} Mean: {}".format(cv_result["test_recall"], cv_result["test_recall"].min(), cv_result["test_recall"].max(), cv_result["test_recall"].mean())))

display(Markdown("**Validation Performance: with Recall Values:**"))
plt.figure(figsize=(10, 7))
for name, model in USDefaultModels:
  model.fit(XTrain, YTrain)
  YPred = model.predict(XValid)
  accuracy = accuracy_score(YValid, YPred)
  recall = recall_score(YValid, YPred)
  precision = precision_score(YValid, YPred)
  f1 = f1_score(YValid, YPred)
  AllModels.append(ModelPerformanceStats(name, accuracy, recall, precision, f1))
  display(Markdown("- Model Name: {} Recall: {}".format(name, recall)))
  fpr, tpr, thresholds = roc_curve(YValid, YPred)
  roc_auc = auc(fpr, tpr)
  plt.plot(fpr, tpr, label=f"{name} (AUC = {roc_auc:.2f})")

# Plot settings
plt.plot([0, 1], [0, 1], color='navy', linestyle='--')  # Diagonal line for random guessing
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate (FPR)')
plt.ylabel('True Positive Rate (TPR)')
plt.title('Receiver Operating Characteristic (ROC) Curve')
plt.legend(loc='lower right')
plt.show()

# Plotting boxplots for CV scores of all models defined above
fig = plt.figure(figsize=(10, 7))

fig.suptitle("CV Comparison")
ax = fig.add_subplot(111)

plt.boxplot(USDefaultModelResults)
ax.set_xticklabels(USDefaultModelNames)

plt.show()

# Tune default Adaptive Boost model
ABCTunedModel = Pipeline([("ABCScaler", StandardScaler()), ("ABCTuned", AdaBoostClassifier(base_estimator = DecisionTreeClassifier(), random_state = 1))])  # Set base_estimator explicitly])

# Grid parameters
param_grid = {"ABCTuned__n_estimators": [100, 150, 200], "ABCTuned__learning_rate": [0.01 ,0.1], "ABCTuned__base_estimator__max_depth": [2, 3]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize GridSearchCV
ABCTunedGrid = GridSearchCV(ABCTunedModel, param_grid = param_grid, scoring = scorer, cv = 5)
# Fit the model
ABCTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('ABCScaler', StandardScaler()),
                                       ('ABCTuned',
                                        AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                           random_state=1))]),
             param_grid={'ABCTuned__base_estimator__max_depth': [2, 3],
                         'ABCTuned__learning_rate': [0.01, 0.1],
                         'ABCTuned__n_estimators': [100, 150, 200]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('ABCScaler', StandardScaler()),
                                       ('ABCTuned',
                                        AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                           random_state=1))]),
             param_grid={'ABCTuned__base_estimator__max_depth': [2, 3],
                         'ABCTuned__learning_rate': [0.01, 0.1],
                         'ABCTuned__n_estimators': [100, 150, 200]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('ABCScaler', StandardScaler()),
                ('ABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                    random_state=1))])

StandardScaler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=1)

DecisionTreeClassifier()

DecisionTreeClassifier()

# Print the best parameters and score
print("Best parameters found by GridSearchCV:", ABCTunedGrid.best_params_)
print("Best cross-validation score (recall):", ABCTunedGrid.best_score_)

# Construct the model with best estimator
ABCTunedGridBest = ABCTunedGrid.best_estimator_
ABCTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'ABCTuned__base_estimator__max_depth': 3, 'ABCTuned__learning_rate': 0.1, 'ABCTuned__n_estimators': 200}
Best cross-validation score (recall): 0.8665071770334928

Pipeline(steps=[('ABCScaler', StandardScaler()),
                ('ABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                                    learning_rate=0.1, n_estimators=200,
                                    random_state=1))])

Pipeline(steps=[('ABCScaler', StandardScaler()),
                ('ABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                                    learning_rate=0.1, n_estimators=200,
                                    random_state=1))])

StandardScaler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                   learning_rate=0.1, n_estimators=200, random_state=1)

DecisionTreeClassifier(max_depth=3)

DecisionTreeClassifier(max_depth=3)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(ABCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(ABCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(ABCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(ABCTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(ABCTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("ABCTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune default Adaptive Boost model
ABCTunedModel = Pipeline([("ABCScaler", StandardScaler()), ("ABCTuned", AdaBoostClassifier(base_estimator = DecisionTreeClassifier(), random_state = 1))])

param_dist = {"ABCTuned__n_estimators": randint(50, 200), "ABCTuned__learning_rate": np.logspace(-2, 0, 10), "ABCTuned__base_estimator__max_depth": randint(2, 4)}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

ABCTunedRandom = RandomizedSearchCV(ABCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1)

# Fit the model
ABCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('ABCScaler', StandardScaler()),
                                             ('ABCTuned',
                                              AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                                 random_state=1))]),
                   param_distributions={'ABCTuned__base_estimator__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c327ac400>,
                                        'ABCTuned__learning_rate': array([0.01      , 0.01668101, 0.02782559, 0.04641589, 0.07742637,
       0.12915497, 0.21544347, 0.35938137, 0.59948425, 1.        ]),
                                        'ABCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31dcf910>},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('ABCScaler', StandardScaler()),
                                             ('ABCTuned',
                                              AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                                 random_state=1))]),
                   param_distributions={'ABCTuned__base_estimator__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c327ac400>,
                                        'ABCTuned__learning_rate': array([0.01      , 0.01668101, 0.02782559, 0.04641589, 0.07742637,
       0.12915497, 0.21544347, 0.35938137, 0.59948425, 1.        ]),
                                        'ABCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31dcf910>},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('ABCScaler', StandardScaler()),
                ('ABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                    random_state=1))])

StandardScaler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=1)

DecisionTreeClassifier()

DecisionTreeClassifier()

# Print the best parameters and best score
print("Best parameters found by RandomizedSearchCV:", ABCTunedRandom.best_params_)
print("Best cross-validation score (recall):", ABCTunedRandom.best_score_)

ABCTunedRandomBest = ABCTunedRandom.best_estimator_
ABCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'ABCTuned__base_estimator__max_depth': 2, 'ABCTuned__learning_rate': 0.3593813663804626, 'ABCTuned__n_estimators': 184}
Best cross-validation score (recall): 0.8732425469267575

Pipeline(steps=[('ABCScaler', StandardScaler()),
                ('ABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                                    learning_rate=0.3593813663804626,
                                    n_estimators=184, random_state=1))])

Pipeline(steps=[('ABCScaler', StandardScaler()),
                ('ABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                                    learning_rate=0.3593813663804626,
                                    n_estimators=184, random_state=1))])

StandardScaler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                   learning_rate=0.3593813663804626, n_estimators=184,
                   random_state=1)

DecisionTreeClassifier(max_depth=2)

DecisionTreeClassifier(max_depth=2)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(ABCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(ABCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(ABCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(ABCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(ABCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("ABCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Adaptive Boost model for over sampled data
OSABCTunedModel = Pipeline([("OSABCScaler", StandardScaler()), ("OSABCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSABCTuned", AdaBoostClassifier(base_estimator = DecisionTreeClassifier(), random_state = 1))])

# Grid parameters
param_grid = {"OSABCTuned__n_estimators": [100, 150, 200], "OSABCTuned__learning_rate": [0.01 ,0.1], "OSABCTuned__base_estimator__max_depth": [2, 3]
}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize GridSearchCV
OSABCTunedGrid = GridSearchCV(OSABCTunedModel, param_grid = param_grid, scoring = scorer, cv = 5)

# Fit the model
OSABCTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('OSABCScaler', StandardScaler()),
                                       ('OSABCSmote', SMOTE(random_state=1)),
                                       ('OSABCTuned',
                                        AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                           random_state=1))]),
             param_grid={'OSABCTuned__base_estimator__max_depth': [2, 3],
                         'OSABCTuned__learning_rate': [0.01, 0.1],
                         'OSABCTuned__n_estimators': [100, 150, 200]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('OSABCScaler', StandardScaler()),
                                       ('OSABCSmote', SMOTE(random_state=1)),
                                       ('OSABCTuned',
                                        AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                           random_state=1))]),
             param_grid={'OSABCTuned__base_estimator__max_depth': [2, 3],
                         'OSABCTuned__learning_rate': [0.01, 0.1],
                         'OSABCTuned__n_estimators': [100, 150, 200]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('OSABCScaler', StandardScaler()),
                ('OSABCSmote', SMOTE(random_state=1)),
                ('OSABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                    random_state=1))])

StandardScaler()

SMOTE(random_state=1)

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=1)

DecisionTreeClassifier()

DecisionTreeClassifier()

# Print the best parameters and score
print("Best parameters found by GridSearchCV:", OSABCTunedGrid.best_params_)
print("Best cross-validation score (recall):", OSABCTunedGrid.best_score_)

# Train a model with the best estimator parameters
OSABCTunedGridBest = OSABCTunedGrid.best_estimator_
OSABCTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'OSABCTuned__base_estimator__max_depth': 3, 'OSABCTuned__learning_rate': 0.01, 'OSABCTuned__n_estimators': 100}
Best cross-validation score (recall): 0.8760627530364372

Pipeline(steps=[('OSABCScaler', StandardScaler()),
                ('OSABCSmote', SMOTE(random_state=1)),
                ('OSABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                                    learning_rate=0.01, n_estimators=100,
                                    random_state=1))])

Pipeline(steps=[('OSABCScaler', StandardScaler()),
                ('OSABCSmote', SMOTE(random_state=1)),
                ('OSABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                                    learning_rate=0.01, n_estimators=100,
                                    random_state=1))])

StandardScaler()

SMOTE(random_state=1)

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                   learning_rate=0.01, n_estimators=100, random_state=1)

DecisionTreeClassifier(max_depth=3)

DecisionTreeClassifier(max_depth=3)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(OSABCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(OSABCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(OSABCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(OSABCTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(OSABCTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("OSABCTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Adaptive Boost model for over sampled data
OSABCTunedModel = Pipeline([("OSABCScaler", StandardScaler()), ("OSABCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSABCTuned", AdaBoostClassifier(base_estimator = DecisionTreeClassifier(), random_state = 1))])

param_dist = {"OSABCTuned__n_estimators": randint(50, 200), "OSABCTuned__learning_rate": np.logspace(-2, 0, 10), "OSABCTuned__base_estimator__max_depth": randint(2, 4)}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

OSABCTunedRandom = RandomizedSearchCV(OSABCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1)

# Fit the model
OSABCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('OSABCScaler', StandardScaler()),
                                             ('OSABCSmote',
                                              SMOTE(random_state=1)),
                                             ('OSABCTuned',
                                              AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                                 random_state=1))]),
                   param_distributions={'OSABCTuned__base_estimator__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c32f4b550>,
                                        'OSABCTuned__learning_rate': array([0.01      , 0.01668101, 0.02782559, 0.04641589, 0.07742637,
       0.12915497, 0.21544347, 0.35938137, 0.59948425, 1.        ]),
                                        'OSABCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31d48c10>},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('OSABCScaler', StandardScaler()),
                                             ('OSABCSmote',
                                              SMOTE(random_state=1)),
                                             ('OSABCTuned',
                                              AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                                 random_state=1))]),
                   param_distributions={'OSABCTuned__base_estimator__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c32f4b550>,
                                        'OSABCTuned__learning_rate': array([0.01      , 0.01668101, 0.02782559, 0.04641589, 0.07742637,
       0.12915497, 0.21544347, 0.35938137, 0.59948425, 1.        ]),
                                        'OSABCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31d48c10>},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('OSABCScaler', StandardScaler()),
                ('OSABCSmote', SMOTE(random_state=1)),
                ('OSABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                    random_state=1))])

StandardScaler()

SMOTE(random_state=1)

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=1)

DecisionTreeClassifier()

DecisionTreeClassifier()

# Print the best parameters and best score
print("Best parameters found by RandomizedSearchCV:", OSABCTunedRandom.best_params_)
print("Best cross-validation score (accuracy):", OSABCTunedRandom.best_score_)

OSABCTunedRandomBest = OSABCTunedRandom.best_estimator_
OSABCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'OSABCTuned__base_estimator__max_depth': 2, 'OSABCTuned__learning_rate': 0.3593813663804626, 'OSABCTuned__n_estimators': 184}
Best cross-validation score (accuracy): 0.8751656238498345

Pipeline(steps=[('OSABCScaler', StandardScaler()),
                ('OSABCSmote', SMOTE(random_state=1)),
                ('OSABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                                    learning_rate=0.3593813663804626,
                                    n_estimators=184, random_state=1))])

Pipeline(steps=[('OSABCScaler', StandardScaler()),
                ('OSABCSmote', SMOTE(random_state=1)),
                ('OSABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                                    learning_rate=0.3593813663804626,
                                    n_estimators=184, random_state=1))])

StandardScaler()

SMOTE(random_state=1)

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                   learning_rate=0.3593813663804626, n_estimators=184,
                   random_state=1)

DecisionTreeClassifier(max_depth=2)

DecisionTreeClassifier(max_depth=2)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(OSABCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(OSABCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(OSABCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(OSABCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(OSABCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("OSABCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Adaptive Boost model for under sampled data
USABCTunedModel = Pipeline([("USABCScaler", StandardScaler()), ("USABCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USABCTuned", AdaBoostClassifier(base_estimator = DecisionTreeClassifier(), random_state = 1))])
# Grid parameters
param_grid = {"USABCTuned__n_estimators": [100, 150, 200], "USABCTuned__learning_rate": [0.01 ,0.1], "USABCTuned__base_estimator__max_depth": [2, 3]
}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize GridSearchCV
USABCTunedGrid = GridSearchCV(USABCTunedModel, param_grid = param_grid, scoring = scorer, cv = 5)

# Fit the model
USABCTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('USABCScaler', StandardScaler()),
                                       ('USABCSmote', RandomUnderSampler()),
                                       ('USABCTuned',
                                        AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                           random_state=1))]),
             param_grid={'USABCTuned__base_estimator__max_depth': [2, 3],
                         'USABCTuned__learning_rate': [0.01, 0.1],
                         'USABCTuned__n_estimators': [100, 150, 200]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('USABCScaler', StandardScaler()),
                                       ('USABCSmote', RandomUnderSampler()),
                                       ('USABCTuned',
                                        AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                           random_state=1))]),
             param_grid={'USABCTuned__base_estimator__max_depth': [2, 3],
                         'USABCTuned__learning_rate': [0.01, 0.1],
                         'USABCTuned__n_estimators': [100, 150, 200]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('USABCScaler', StandardScaler()),
                ('USABCSmote', RandomUnderSampler()),
                ('USABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                    random_state=1))])

StandardScaler()

RandomUnderSampler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=1)

DecisionTreeClassifier()

DecisionTreeClassifier()

# Print the best parameters and score
print("Best parameters found by GridSearchCV:", USABCTunedGrid.best_params_)
print("Best cross-validation score (recall):", USABCTunedGrid.best_score_)

USABCTunedGridBest = USABCTunedGrid.best_estimator_
USABCTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'USABCTuned__base_estimator__max_depth': 2, 'USABCTuned__learning_rate': 0.1, 'USABCTuned__n_estimators': 200}
Best cross-validation score (recall): 0.9558152373941848

Pipeline(steps=[('USABCScaler', StandardScaler()),
                ('USABCSmote', RandomUnderSampler()),
                ('USABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                                    learning_rate=0.1, n_estimators=200,
                                    random_state=1))])

Pipeline(steps=[('USABCScaler', StandardScaler()),
                ('USABCSmote', RandomUnderSampler()),
                ('USABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                                    learning_rate=0.1, n_estimators=200,
                                    random_state=1))])

StandardScaler()

RandomUnderSampler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=2),
                   learning_rate=0.1, n_estimators=200, random_state=1)

DecisionTreeClassifier(max_depth=2)

DecisionTreeClassifier(max_depth=2)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USABCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USABCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USABCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USABCTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(USABCTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("USABCTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Adaptive Boost model for under sampled data
USABCTunedModel = Pipeline([("USABCScaler", StandardScaler()), ("USABCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USABCTuned", AdaBoostClassifier(base_estimator = DecisionTreeClassifier(), random_state = 1))])

param_dist = {"USABCTuned__n_estimators": randint(50, 200), "USABCTuned__learning_rate": np.logspace(-2, 0, 10), "USABCTuned__base_estimator__max_depth": randint(2, 4)}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

USABCTunedRandom = RandomizedSearchCV(USABCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1)

# Fit the model
USABCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('USABCScaler', StandardScaler()),
                                             ('USABCSmote',
                                              RandomUnderSampler()),
                                             ('USABCTuned',
                                              AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                                 random_state=1))]),
                   param_distributions={'USABCTuned__base_estimator__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c3120d000>,
                                        'USABCTuned__learning_rate': array([0.01      , 0.01668101, 0.02782559, 0.04641589, 0.07742637,
       0.12915497, 0.21544347, 0.35938137, 0.59948425, 1.        ]),
                                        'USABCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31084310>},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('USABCScaler', StandardScaler()),
                                             ('USABCSmote',
                                              RandomUnderSampler()),
                                             ('USABCTuned',
                                              AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                                                 random_state=1))]),
                   param_distributions={'USABCTuned__base_estimator__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c3120d000>,
                                        'USABCTuned__learning_rate': array([0.01      , 0.01668101, 0.02782559, 0.04641589, 0.07742637,
       0.12915497, 0.21544347, 0.35938137, 0.59948425, 1.        ]),
                                        'USABCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31084310>},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('USABCScaler', StandardScaler()),
                ('USABCSmote', RandomUnderSampler()),
                ('USABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(),
                                    random_state=1))])

StandardScaler()

RandomUnderSampler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(), random_state=1)

DecisionTreeClassifier()

DecisionTreeClassifier()

# Print the best parameters and best score
print("Best parameters found by RandomizedSearchCV:", USABCTunedRandom.best_params_)
print("Best cross-validation score (accuracy):", USABCTunedRandom.best_score_)

USABCTunedRandomBest = USABCTunedRandom.best_estimator_
USABCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'USABCTuned__base_estimator__max_depth': 3, 'USABCTuned__learning_rate': 0.5994842503189409, 'USABCTuned__n_estimators': 187}
Best cross-validation score (accuracy): 0.9519874861980124

Pipeline(steps=[('USABCScaler', StandardScaler()),
                ('USABCSmote', RandomUnderSampler()),
                ('USABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                                    learning_rate=0.5994842503189409,
                                    n_estimators=187, random_state=1))])

Pipeline(steps=[('USABCScaler', StandardScaler()),
                ('USABCSmote', RandomUnderSampler()),
                ('USABCTuned',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                                    learning_rate=0.5994842503189409,
                                    n_estimators=187, random_state=1))])

StandardScaler()

RandomUnderSampler()

AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3),
                   learning_rate=0.5994842503189409, n_estimators=187,
                   random_state=1)

DecisionTreeClassifier(max_depth=3)

DecisionTreeClassifier(max_depth=3)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USABCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USABCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USABCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USABCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(USABCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("USABCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Gradient Boost model
GBCTunedModel = Pipeline([("GBCScaler", StandardScaler()), ("GBCTuned", GradientBoostingClassifier(random_state = 1))])

param_grid = {"GBCTuned__n_estimators": [50, 150], "GBCTuned__learning_rate": [0.01, 0.5], "GBCTuned__max_depth": [4, 5], "GBCTuned__subsample": [0.6, 0.8]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

GBCTunedGrid = GridSearchCV(GBCTunedModel, param_grid = param_grid, cv = 5, scoring = scorer, n_jobs = -1)

GBCTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('GBCScaler', StandardScaler()),
                                       ('GBCTuned',
                                        GradientBoostingClassifier(random_state=1))]),
             n_jobs=-1,
             param_grid={'GBCTuned__learning_rate': [0.01, 0.5],
                         'GBCTuned__max_depth': [4, 5],
                         'GBCTuned__n_estimators': [50, 150],
                         'GBCTuned__subsample': [0.6, 0.8]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('GBCScaler', StandardScaler()),
                                       ('GBCTuned',
                                        GradientBoostingClassifier(random_state=1))]),
             n_jobs=-1,
             param_grid={'GBCTuned__learning_rate': [0.01, 0.5],
                         'GBCTuned__max_depth': [4, 5],
                         'GBCTuned__n_estimators': [50, 150],
                         'GBCTuned__subsample': [0.6, 0.8]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('GBCScaler', StandardScaler()),
                ('GBCTuned', GradientBoostingClassifier(random_state=1))])

StandardScaler()

GradientBoostingClassifier(random_state=1)

# Print the best parameters and score for GridSearchCV
print("Best parameters found by GridSearchCV:", GBCTunedGrid.best_params_)
print("Best cross-validation score (recall):", GBCTunedGrid.best_score_)

#Get the model with best estimator
GBCTunedGridBest = GBCTunedGrid.best_estimator_
GBCTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'GBCTuned__learning_rate': 0.5, 'GBCTuned__max_depth': 4, 'GBCTuned__n_estimators': 150, 'GBCTuned__subsample': 0.8}
Best cross-validation score (recall): 0.8588194700036805

Pipeline(steps=[('GBCScaler', StandardScaler()),
                ('GBCTuned',
                 GradientBoostingClassifier(learning_rate=0.5, max_depth=4,
                                            n_estimators=150, random_state=1,
                                            subsample=0.8))])

Pipeline(steps=[('GBCScaler', StandardScaler()),
                ('GBCTuned',
                 GradientBoostingClassifier(learning_rate=0.5, max_depth=4,
                                            n_estimators=150, random_state=1,
                                            subsample=0.8))])

StandardScaler()

GradientBoostingClassifier(learning_rate=0.5, max_depth=4, n_estimators=150,
                           random_state=1, subsample=0.8)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(GBCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(GBCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(GBCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(GBCTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(GBCTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("GBCTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Gradient Boost model
GBCTunedModel = Pipeline([("GBCScaler", StandardScaler()), ("GBCTuned", GradientBoostingClassifier(random_state = 1))])

param_dist = {"GBCTuned__n_estimators": randint(50, 200), "GBCTuned__learning_rate": np.logspace(-3, 0, 10), "GBCTuned__max_depth": randint(2, 6), "GBCTuned__subsample": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# 5b. Run RandomizedSearchCV
GBCTunedRandom = RandomizedSearchCV(GBCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1, n_jobs=-1)
GBCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('GBCScaler', StandardScaler()),
                                             ('GBCTuned',
                                              GradientBoostingClassifier(random_state=1))]),
                   n_jobs=-1,
                   param_distributions={'GBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'GBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c3110dab0>,
                                        'GBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31102aa0>,
                                        'GBCTuned__subsample': [0.6, 0.8, 1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('GBCScaler', StandardScaler()),
                                             ('GBCTuned',
                                              GradientBoostingClassifier(random_state=1))]),
                   n_jobs=-1,
                   param_distributions={'GBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'GBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c3110dab0>,
                                        'GBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c31102aa0>,
                                        'GBCTuned__subsample': [0.6, 0.8, 1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('GBCScaler', StandardScaler()),
                ('GBCTuned', GradientBoostingClassifier(random_state=1))])

StandardScaler()

GradientBoostingClassifier(random_state=1)

# Print the best parameters and score for RandomizedSearchCV
print("Best parameters found by RandomizedSearchCV:", GBCTunedRandom.best_params_)
print("Best cross-validation score (accuracy):", GBCTunedRandom.best_score_)

GBCTunedRandomBest = GBCTunedRandom.best_estimator_
GBCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'GBCTuned__learning_rate': 0.046415888336127774, 'GBCTuned__max_depth': 5, 'GBCTuned__n_estimators': 190, 'GBCTuned__subsample': 0.6}
Best cross-validation score (accuracy): 0.8530456385719543

Pipeline(steps=[('GBCScaler', StandardScaler()),
                ('GBCTuned',
                 GradientBoostingClassifier(learning_rate=0.046415888336127774,
                                            max_depth=5, n_estimators=190,
                                            random_state=1, subsample=0.6))])

Pipeline(steps=[('GBCScaler', StandardScaler()),
                ('GBCTuned',
                 GradientBoostingClassifier(learning_rate=0.046415888336127774,
                                            max_depth=5, n_estimators=190,
                                            random_state=1, subsample=0.6))])

StandardScaler()

GradientBoostingClassifier(learning_rate=0.046415888336127774, max_depth=5,
                           n_estimators=190, random_state=1, subsample=0.6)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(GBCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(GBCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(GBCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(GBCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(GBCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("GBCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Gradient Boost Algorithm with over sampled data
OSGBCTunedModel = Pipeline([("OSGBCScaler", StandardScaler()), ("OSGBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSGBCTuned", GradientBoostingClassifier(random_state = 1))])

param_grid = {"OSGBCTuned__n_estimators": [50, 150], "OSGBCTuned__learning_rate": [0.01, 0.5], "OSGBCTuned__max_depth": [4, 5], "OSGBCTuned__subsample": [0.6, 0.8]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

OSGBCTunedGrid = GridSearchCV(OSGBCTunedModel, param_grid = param_grid, cv = 5, scoring = scorer, n_jobs = -1)

OSGBCTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                                       ('OSGBCSmote', SMOTE(random_state=1)),
                                       ('OSGBCTuned',
                                        GradientBoostingClassifier(random_state=1))]),
             n_jobs=-1,
             param_grid={'OSGBCTuned__learning_rate': [0.01, 0.5],
                         'OSGBCTuned__max_depth': [4, 5],
                         'OSGBCTuned__n_estimators': [50, 150],
                         'OSGBCTuned__subsample': [0.6, 0.8]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                                       ('OSGBCSmote', SMOTE(random_state=1)),
                                       ('OSGBCTuned',
                                        GradientBoostingClassifier(random_state=1))]),
             n_jobs=-1,
             param_grid={'OSGBCTuned__learning_rate': [0.01, 0.5],
                         'OSGBCTuned__max_depth': [4, 5],
                         'OSGBCTuned__n_estimators': [50, 150],
                         'OSGBCTuned__subsample': [0.6, 0.8]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                ('OSGBCSmote', SMOTE(random_state=1)),
                ('OSGBCTuned', GradientBoostingClassifier(random_state=1))])

StandardScaler()

SMOTE(random_state=1)

GradientBoostingClassifier(random_state=1)

# Print the best parameters and score for GridSearchCV
print("Best parameters found by GridSearchCV:", OSGBCTunedGrid.best_params_)
print("Best cross-validation score (recall):", OSGBCTunedGrid.best_score_)

#Tune the model with best estimator
OSGBCTunedGridBest = OSGBCTunedGrid.best_estimator_
OSGBCTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'OSGBCTuned__learning_rate': 0.01, 'OSGBCTuned__max_depth': 4, 'OSGBCTuned__n_estimators': 150, 'OSGBCTuned__subsample': 0.6}
Best cross-validation score (recall): 0.9029812292970189

Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                ('OSGBCSmote', SMOTE(random_state=1)),
                ('OSGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.01, max_depth=4,
                                            n_estimators=150, random_state=1,
                                            subsample=0.6))])

Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                ('OSGBCSmote', SMOTE(random_state=1)),
                ('OSGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.01, max_depth=4,
                                            n_estimators=150, random_state=1,
                                            subsample=0.6))])

StandardScaler()

SMOTE(random_state=1)

GradientBoostingClassifier(learning_rate=0.01, max_depth=4, n_estimators=150,
                           random_state=1, subsample=0.6)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(OSGBCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(OSGBCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(OSGBCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(OSGBCTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(OSGBCTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("OSGBCTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Gradient Boost Algorithm with over sampled data
OSGBCTunedModel = Pipeline([("OSGBCScaler", StandardScaler()), ("OSGBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSGBCTuned", GradientBoostingClassifier(random_state = 1))])

param_dist = {"OSGBCTuned__n_estimators": randint(50, 200), "OSGBCTuned__learning_rate": np.logspace(-3, 0, 10), "OSGBCTuned__max_depth": randint(2, 6), "OSGBCTuned__subsample": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# 5b. Run RandomizedSearchCV
OSGBCTunedRandom = RandomizedSearchCV(OSGBCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1, n_jobs=-1)
OSGBCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                                             ('OSGBCSmote',
                                              SMOTE(random_state=1)),
                                             ('OSGBCTuned',
                                              GradientBoostingClassifier(random_state=1))]),
                   n_jobs=-1,
                   param_distributions={'OSGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'OSGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c30492170>,
                                        'OSGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2fc32ef0>,
                                        'OSGBCTuned__subsample': [0.6, 0.8,
                                                                  1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                                             ('OSGBCSmote',
                                              SMOTE(random_state=1)),
                                             ('OSGBCTuned',
                                              GradientBoostingClassifier(random_state=1))]),
                   n_jobs=-1,
                   param_distributions={'OSGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'OSGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c30492170>,
                                        'OSGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2fc32ef0>,
                                        'OSGBCTuned__subsample': [0.6, 0.8,
                                                                  1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                ('OSGBCSmote', SMOTE(random_state=1)),
                ('OSGBCTuned', GradientBoostingClassifier(random_state=1))])

StandardScaler()

SMOTE(random_state=1)

GradientBoostingClassifier(random_state=1)

# Print the best parameters and score for RandomizedSearchCV
print("Best parameters found by RandomizedSearchCV:", OSGBCTunedRandom.best_params_)
print("Best cross-validation score (accuracy):", OSGBCTunedRandom.best_score_)

OSGBCTunedRandomBest = OSGBCTunedRandom.best_estimator_
OSGBCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'OSGBCTuned__learning_rate': 0.046415888336127774, 'OSGBCTuned__max_depth': 5, 'OSGBCTuned__n_estimators': 190, 'OSGBCTuned__subsample': 0.6}
Best cross-validation score (accuracy): 0.8962596613912404

Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                ('OSGBCSmote', SMOTE(random_state=1)),
                ('OSGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.046415888336127774,
                                            max_depth=5, n_estimators=190,
                                            random_state=1, subsample=0.6))])

Pipeline(steps=[('OSGBCScaler', StandardScaler()),
                ('OSGBCSmote', SMOTE(random_state=1)),
                ('OSGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.046415888336127774,
                                            max_depth=5, n_estimators=190,
                                            random_state=1, subsample=0.6))])

StandardScaler()

SMOTE(random_state=1)

GradientBoostingClassifier(learning_rate=0.046415888336127774, max_depth=5,
                           n_estimators=190, random_state=1, subsample=0.6)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(OSGBCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(OSGBCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(OSGBCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(OSGBCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(OSGBCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("OSGBCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the Gradient Boost Algorithm with under sampled data
USGBCTunedModel = Pipeline([("USGBCScaler", StandardScaler()), ("USGBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USGBCTuned", GradientBoostingClassifier(random_state = 1))])

param_grid = {"USGBCTuned__n_estimators": [50, 150], "USGBCTuned__learning_rate": [0.01, 0.5], "USGBCTuned__max_depth": [4, 5], "USGBCTuned__subsample": [0.6, 0.8]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

USGBCTunedGrid = GridSearchCV(USGBCTunedModel, param_grid = param_grid, cv = 5, scoring = scorer, n_jobs = -1)

USGBCTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('USGBCScaler', StandardScaler()),
                                       ('USGBCSmote', RandomUnderSampler()),
                                       ('USGBCTuned',
                                        GradientBoostingClassifier(random_state=1))]),
             n_jobs=-1,
             param_grid={'USGBCTuned__learning_rate': [0.01, 0.5],
                         'USGBCTuned__max_depth': [4, 5],
                         'USGBCTuned__n_estimators': [50, 150],
                         'USGBCTuned__subsample': [0.6, 0.8]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('USGBCScaler', StandardScaler()),
                                       ('USGBCSmote', RandomUnderSampler()),
                                       ('USGBCTuned',
                                        GradientBoostingClassifier(random_state=1))]),
             n_jobs=-1,
             param_grid={'USGBCTuned__learning_rate': [0.01, 0.5],
                         'USGBCTuned__max_depth': [4, 5],
                         'USGBCTuned__n_estimators': [50, 150],
                         'USGBCTuned__subsample': [0.6, 0.8]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('USGBCScaler', StandardScaler()),
                ('USGBCSmote', RandomUnderSampler()),
                ('USGBCTuned', GradientBoostingClassifier(random_state=1))])

StandardScaler()

RandomUnderSampler()

GradientBoostingClassifier(random_state=1)

# Print the best parameters and score for GridSearchCV
print("Best parameters found by GridSearchCV:", USGBCTunedGrid.best_params_)
print("Best cross-validation score (recall):", USGBCTunedGrid.best_score_)

USGBCTunedGridBest = USGBCTunedGrid.best_estimator_
USGBCTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'USGBCTuned__learning_rate': 0.5, 'USGBCTuned__max_depth': 4, 'USGBCTuned__n_estimators': 150, 'USGBCTuned__subsample': 0.8}
Best cross-validation score (recall): 0.9510213470739787

Pipeline(steps=[('USGBCScaler', StandardScaler()),
                ('USGBCSmote', RandomUnderSampler()),
                ('USGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.5, max_depth=4,
                                            n_estimators=150, random_state=1,
                                            subsample=0.8))])

Pipeline(steps=[('USGBCScaler', StandardScaler()),
                ('USGBCSmote', RandomUnderSampler()),
                ('USGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.5, max_depth=4,
                                            n_estimators=150, random_state=1,
                                            subsample=0.8))])

StandardScaler()

RandomUnderSampler()

GradientBoostingClassifier(learning_rate=0.5, max_depth=4, n_estimators=150,
                           random_state=1, subsample=0.8)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USGBCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USGBCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USGBCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USGBCTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(USGBCTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("USGBCTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

USGBCTunedModel = Pipeline([("USGBCScaler", StandardScaler()), ("USGBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USGBCTuned", GradientBoostingClassifier(random_state = 1))])

param_dist = {"USGBCTuned__n_estimators": randint(50, 200), "USGBCTuned__learning_rate": np.logspace(-3, 0, 10), "USGBCTuned__max_depth": randint(2, 6), "USGBCTuned__subsample": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# 5b. Run RandomizedSearchCV
USGBCTunedRandom = RandomizedSearchCV(USGBCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1, n_jobs=-1)
USGBCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('USGBCScaler', StandardScaler()),
                                             ('USGBCSmote',
                                              RandomUnderSampler()),
                                             ('USGBCTuned',
                                              GradientBoostingClassifier(random_state=1))]),
                   n_jobs=-1,
                   param_distributions={'USGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'USGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2fc30b80>,
                                        'USGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f9ab3d0>,
                                        'USGBCTuned__subsample': [0.6, 0.8,
                                                                  1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('USGBCScaler', StandardScaler()),
                                             ('USGBCSmote',
                                              RandomUnderSampler()),
                                             ('USGBCTuned',
                                              GradientBoostingClassifier(random_state=1))]),
                   n_jobs=-1,
                   param_distributions={'USGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'USGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2fc30b80>,
                                        'USGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f9ab3d0>,
                                        'USGBCTuned__subsample': [0.6, 0.8,
                                                                  1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('USGBCScaler', StandardScaler()),
                ('USGBCSmote', RandomUnderSampler()),
                ('USGBCTuned', GradientBoostingClassifier(random_state=1))])

StandardScaler()

RandomUnderSampler()

GradientBoostingClassifier(random_state=1)

# Print the best parameters and score for RandomizedSearchCV
print("Best parameters found by RandomizedSearchCV:", USGBCTunedRandom.best_params_)
print("Best cross-validation score (accuracy):", USGBCTunedRandom.best_score_)

USGBCTunedRandomBest = USGBCTunedRandom.best_estimator_
USGBCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'USGBCTuned__learning_rate': 0.046415888336127774, 'USGBCTuned__max_depth': 5, 'USGBCTuned__n_estimators': 190, 'USGBCTuned__subsample': 0.6}
Best cross-validation score (accuracy): 0.9548444976076554

Pipeline(steps=[('USGBCScaler', StandardScaler()),
                ('USGBCSmote', RandomUnderSampler()),
                ('USGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.046415888336127774,
                                            max_depth=5, n_estimators=190,
                                            random_state=1, subsample=0.6))])

Pipeline(steps=[('USGBCScaler', StandardScaler()),
                ('USGBCSmote', RandomUnderSampler()),
                ('USGBCTuned',
                 GradientBoostingClassifier(learning_rate=0.046415888336127774,
                                            max_depth=5, n_estimators=190,
                                            random_state=1, subsample=0.6))])

StandardScaler()

RandomUnderSampler()

GradientBoostingClassifier(learning_rate=0.046415888336127774, max_depth=5,
                           n_estimators=190, random_state=1, subsample=0.6)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USGBCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USGBCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USGBCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USGBCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(USGBCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("USGBCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune XGBoost Model
XGBCTunedModel = Pipeline([("XGBCScaler", StandardScaler()), ("XGBCTuned", XGBClassifier(random_state = 1))])

# Define hyperparameter grid for GridSearchCV
param_grid = {"XGBCTuned__n_estimators": [50, 100, 200], "XGBCTuned__learning_rate": [0.01, 0.1, 0.3], "XGBCTuned__max_depth": [3, 5, 7], "XGBCTuned__subsample": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize GridSearchCV
XGBTunedGrid = GridSearchCV(XGBCTunedModel, param_grid = param_grid, cv=5, scoring = scorer, n_jobs = -1)

# Fit the model
XGBTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('XGBCScaler', StandardScaler()),
                                       ('XGBCTuned',
                                        XGBClassifier(base_score=None,
                                                      booster=None,
                                                      callbacks=None,
                                                      colsample_bylevel=None,
                                                      colsample_bynode=None,
                                                      colsample_bytree=None,
                                                      device=None,
                                                      early_stopping_rounds=None,
                                                      enable_categorical=False,
                                                      eval_metric=None,
                                                      feature_types=None,
                                                      gamma=None,
                                                      grow_policy=None,
                                                      importan...
                                                      min_child_weight=None,
                                                      missing=nan,
                                                      monotone_constraints=None,
                                                      multi_strategy=None,
                                                      n_estimators=None,
                                                      n_jobs=None,
                                                      num_parallel_tree=None,
                                                      random_state=1, ...))]),
             n_jobs=-1,
             param_grid={'XGBCTuned__learning_rate': [0.01, 0.1, 0.3],
                         'XGBCTuned__max_depth': [3, 5, 7],
                         'XGBCTuned__n_estimators': [50, 100, 200],
                         'XGBCTuned__subsample': [0.6, 0.8, 1.0]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('XGBCScaler', StandardScaler()),
                                       ('XGBCTuned',
                                        XGBClassifier(base_score=None,
                                                      booster=None,
                                                      callbacks=None,
                                                      colsample_bylevel=None,
                                                      colsample_bynode=None,
                                                      colsample_bytree=None,
                                                      device=None,
                                                      early_stopping_rounds=None,
                                                      enable_categorical=False,
                                                      eval_metric=None,
                                                      feature_types=None,
                                                      gamma=None,
                                                      grow_policy=None,
                                                      importan...
                                                      min_child_weight=None,
                                                      missing=nan,
                                                      monotone_constraints=None,
                                                      multi_strategy=None,
                                                      n_estimators=None,
                                                      n_jobs=None,
                                                      num_parallel_tree=None,
                                                      random_state=1, ...))]),
             n_jobs=-1,
             param_grid={'XGBCTuned__learning_rate': [0.01, 0.1, 0.3],
                         'XGBCTuned__max_depth': [3, 5, 7],
                         'XGBCTuned__n_estimators': [50, 100, 200],
                         'XGBCTuned__subsample': [0.6, 0.8, 1.0]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('XGBCScaler', StandardScaler()),
                ('XGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=None, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Print the best parameters and score for GridSearchCV
print("Best parameters found by GridSearchCV:", XGBTunedGrid.best_params_)
print("Best cross-validation score (recall):", XGBTunedGrid.best_score_)

XGBTunedGridBest = XGBTunedGrid.best_estimator_
XGBTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'XGBCTuned__learning_rate': 0.1, 'XGBCTuned__max_depth': 7, 'XGBCTuned__n_estimators': 200, 'XGBCTuned__subsample': 1.0}
Best cross-validation score (recall): 0.8789887743835113

Pipeline(steps=[('XGBCScaler', StandardScaler()),
                ('XGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.1,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=7, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=200, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

Pipeline(steps=[('XGBCScaler', StandardScaler()),
                ('XGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.1,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=7, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=200, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.1, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=7, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=200, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(XGBTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(XGBTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(XGBTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(XGBTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(XGBTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("XGBTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune XGBoost Model
XGBCTunedModel = Pipeline([("XGBCScaler", StandardScaler()), ("XGBCTuned", XGBClassifier(random_state = 1))])

param_dist = {"XGBCTuned__n_estimators": randint(50, 200), "XGBCTuned__learning_rate": np.logspace(-3, 0, 10),  "XGBCTuned__max_depth": randint(3, 10), "XGBCTuned__subsample": [0.6, 0.8, 1.0], "XGBCTuned__colsample_bytree": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize RandomizedSearchCV
XGBCTunedRandom = RandomizedSearchCV(XGBCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1, n_jobs = -1)

# Fit the model
XGBCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('XGBCScaler', StandardScaler()),
                                             ('XGBCTuned',
                                              XGBClassifier(base_score=None,
                                                            booster=None,
                                                            callbacks=None,
                                                            colsample_bylevel=None,
                                                            colsample_bynode=None,
                                                            colsample_bytree=None,
                                                            device=None,
                                                            early_stopping_rounds=None,
                                                            enable_categorical=False,
                                                            eval_metric=None,
                                                            feature_types=None,
                                                            gamma=None,
                                                            grow_policy=None,
                                                            im...
                                        'XGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'XGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f95a230>,
                                        'XGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f9abcd0>,
                                        'XGBCTuned__subsample': [0.6, 0.8,
                                                                 1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('XGBCScaler', StandardScaler()),
                                             ('XGBCTuned',
                                              XGBClassifier(base_score=None,
                                                            booster=None,
                                                            callbacks=None,
                                                            colsample_bylevel=None,
                                                            colsample_bynode=None,
                                                            colsample_bytree=None,
                                                            device=None,
                                                            early_stopping_rounds=None,
                                                            enable_categorical=False,
                                                            eval_metric=None,
                                                            feature_types=None,
                                                            gamma=None,
                                                            grow_policy=None,
                                                            im...
                                        'XGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'XGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f95a230>,
                                        'XGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f9abcd0>,
                                        'XGBCTuned__subsample': [0.6, 0.8,
                                                                 1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('XGBCScaler', StandardScaler()),
                ('XGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=None, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Print the best parameters and score for RandomizedSearchCV
print("Best parameters found by RandomizedSearchCV:", XGBCTunedRandom.best_params_)
print("Best cross-validation score (recall):", XGBCTunedRandom.best_score_)

XGBCTunedRandomBest = XGBCTunedRandom.best_estimator_
XGBCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'XGBCTuned__colsample_bytree': 0.8, 'XGBCTuned__learning_rate': 0.46415888336127775, 'XGBCTuned__max_depth': 3, 'XGBCTuned__n_estimators': 191, 'XGBCTuned__subsample': 1.0}
Best cross-validation score (recall): 0.8741764814133235

Pipeline(steps=[('XGBCScaler', StandardScaler()),
                ('XGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=0.8, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None,
                               learning_rate=0.46415888336127775, max_bin=None,
                               max_cat_threshold=None, max_cat_to_onehot=None,
                               max_delta_step=None, max_depth=3,
                               max_leaves=None, min_child_weight=None,
                               missing=nan, monotone_constraints=None,
                               multi_strategy=None, n_estimators=191,
                               n_jobs=None, num_parallel_tree=None,
                               random_state=1, ...))])

Pipeline(steps=[('XGBCScaler', StandardScaler()),
                ('XGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=0.8, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None,
                               learning_rate=0.46415888336127775, max_bin=None,
                               max_cat_threshold=None, max_cat_to_onehot=None,
                               max_delta_step=None, max_depth=3,
                               max_leaves=None, min_child_weight=None,
                               missing=nan, monotone_constraints=None,
                               multi_strategy=None, n_estimators=191,
                               n_jobs=None, num_parallel_tree=None,
                               random_state=1, ...))])

StandardScaler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.8, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.46415888336127775,
              max_bin=None, max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=3, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=191, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(XGBCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(XGBCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(XGBCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(XGBCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(XGBCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("XGBCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

OSXGBCTunedModel = Pipeline([("OSXGBCScaler", StandardScaler()), ("OSXGBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSXGBCTuned", XGBClassifier(random_state = 1))])

# Define hyperparameter grid for GridSearchCV
param_grid = {"OSXGBCTuned__n_estimators": [50, 100, 200], "OSXGBCTuned__learning_rate": [0.01, 0.1, 0.3], "OSXGBCTuned__max_depth": [3, 5, 7], "OSXGBCTuned__subsample": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize GridSearchCV
OSXGBTunedGrid = GridSearchCV(OSXGBCTunedModel, param_grid = param_grid, cv=5, scoring = scorer, n_jobs = -1)

# Fit the model
OSXGBTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                                       ('OSXGBCSmote', SMOTE(random_state=1)),
                                       ('OSXGBCTuned',
                                        XGBClassifier(base_score=None,
                                                      booster=None,
                                                      callbacks=None,
                                                      colsample_bylevel=None,
                                                      colsample_bynode=None,
                                                      colsample_bytree=None,
                                                      device=None,
                                                      early_stopping_rounds=None,
                                                      enable_categorical=False,
                                                      eval_metric=None,
                                                      feature_types...
                                                      min_child_weight=None,
                                                      missing=nan,
                                                      monotone_constraints=None,
                                                      multi_strategy=None,
                                                      n_estimators=None,
                                                      n_jobs=None,
                                                      num_parallel_tree=None,
                                                      random_state=1, ...))]),
             n_jobs=-1,
             param_grid={'OSXGBCTuned__learning_rate': [0.01, 0.1, 0.3],
                         'OSXGBCTuned__max_depth': [3, 5, 7],
                         'OSXGBCTuned__n_estimators': [50, 100, 200],
                         'OSXGBCTuned__subsample': [0.6, 0.8, 1.0]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                                       ('OSXGBCSmote', SMOTE(random_state=1)),
                                       ('OSXGBCTuned',
                                        XGBClassifier(base_score=None,
                                                      booster=None,
                                                      callbacks=None,
                                                      colsample_bylevel=None,
                                                      colsample_bynode=None,
                                                      colsample_bytree=None,
                                                      device=None,
                                                      early_stopping_rounds=None,
                                                      enable_categorical=False,
                                                      eval_metric=None,
                                                      feature_types...
                                                      min_child_weight=None,
                                                      missing=nan,
                                                      monotone_constraints=None,
                                                      multi_strategy=None,
                                                      n_estimators=None,
                                                      n_jobs=None,
                                                      num_parallel_tree=None,
                                                      random_state=1, ...))]),
             n_jobs=-1,
             param_grid={'OSXGBCTuned__learning_rate': [0.01, 0.1, 0.3],
                         'OSXGBCTuned__max_depth': [3, 5, 7],
                         'OSXGBCTuned__n_estimators': [50, 100, 200],
                         'OSXGBCTuned__subsample': [0.6, 0.8, 1.0]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                ('OSXGBCSmote', SMOTE(random_state=1)),
                ('OSXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=None, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

SMOTE(random_state=1)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Print the best parameters and score for GridSearchCV
print("Best parameters found by GridSearchCV:", OSXGBTunedGrid.best_params_)
print("Best cross-validation score (recall):", OSXGBTunedGrid.best_score_)

OSXGBTunedGridBest = OSXGBTunedGrid.best_estimator_
OSXGBTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'OSXGBCTuned__learning_rate': 0.01, 'OSXGBCTuned__max_depth': 3, 'OSXGBCTuned__n_estimators': 100, 'OSXGBCTuned__subsample': 0.6}
Best cross-validation score (recall): 0.911635075450865

Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                ('OSXGBCSmote', SMOTE(random_state=1)),
                ('OSXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.01,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=3, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=100, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                ('OSXGBCSmote', SMOTE(random_state=1)),
                ('OSXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.01,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=3, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=100, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

SMOTE(random_state=1)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.01, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=3, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=100, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(OSXGBTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(OSXGBTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(OSXGBTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(OSXGBTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(OSXGBTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("OSXGBTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune XG Boost Oversampled
OSXGBCTunedModel = Pipeline([("OSXGBCScaler", StandardScaler()), ("OSXGBCSmote", SMOTE(sampling_strategy = "auto", k_neighbors = 5, random_state = 1)), ("OSXGBCTuned", XGBClassifier(random_state = 1))])

param_dist = {"OSXGBCTuned__n_estimators": randint(50, 200), "OSXGBCTuned__learning_rate": np.logspace(-3, 0, 10),  "OSXGBCTuned__max_depth": randint(3, 10), "OSXGBCTuned__subsample": [0.6, 0.8, 1.0], "OSXGBCTuned__colsample_bytree": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize RandomizedSearchCV
OSXGBCTunedRandom = RandomizedSearchCV(OSXGBCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1, n_jobs = -1)

# Fit the model
OSXGBCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                                             ('OSXGBCSmote',
                                              SMOTE(random_state=1)),
                                             ('OSXGBCTuned',
                                              XGBClassifier(base_score=None,
                                                            booster=None,
                                                            callbacks=None,
                                                            colsample_bylevel=None,
                                                            colsample_bynode=None,
                                                            colsample_bytree=None,
                                                            device=None,
                                                            early_stopping_rounds=None,
                                                            enable_categorical=False,
                                                            eval_metric=None,
                                                            feature...
                                        'OSXGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'OSXGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f7fc9d0>,
                                        'OSXGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f54fc40>,
                                        'OSXGBCTuned__subsample': [0.6, 0.8,
                                                                   1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                                             ('OSXGBCSmote',
                                              SMOTE(random_state=1)),
                                             ('OSXGBCTuned',
                                              XGBClassifier(base_score=None,
                                                            booster=None,
                                                            callbacks=None,
                                                            colsample_bylevel=None,
                                                            colsample_bynode=None,
                                                            colsample_bytree=None,
                                                            device=None,
                                                            early_stopping_rounds=None,
                                                            enable_categorical=False,
                                                            eval_metric=None,
                                                            feature...
                                        'OSXGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'OSXGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f7fc9d0>,
                                        'OSXGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f54fc40>,
                                        'OSXGBCTuned__subsample': [0.6, 0.8,
                                                                   1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                ('OSXGBCSmote', SMOTE(random_state=1)),
                ('OSXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=None, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

SMOTE(random_state=1)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Print the best parameters and score for RandomizedSearchCV
print("Best parameters found by RandomizedSearchCV:", OSXGBCTunedRandom.best_params_)
print("Best cross-validation score (recall):", OSXGBCTunedRandom.best_score_)

OSXGBCTunedRandomBest = OSXGBCTunedRandom.best_estimator_
OSXGBCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'OSXGBCTuned__colsample_bytree': 0.6, 'OSXGBCTuned__learning_rate': 0.1, 'OSXGBCTuned__max_depth': 4, 'OSXGBCTuned__n_estimators': 70, 'OSXGBCTuned__subsample': 1.0}
Best cross-validation score (recall): 0.8895196908354803

Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                ('OSXGBCSmote', SMOTE(random_state=1)),
                ('OSXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=0.6, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.1,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=4, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=70, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

Pipeline(steps=[('OSXGBCScaler', StandardScaler()),
                ('OSXGBCSmote', SMOTE(random_state=1)),
                ('OSXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=0.6, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.1,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=4, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=70, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

SMOTE(random_state=1)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.6, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.1, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=4, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=70, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(OSXGBCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(OSXGBCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(OSXGBCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(OSXGBCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(OSXGBCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("OSXGBCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Tune the XGBoost with Under sampling
USXGBCTunedModel = Pipeline([("USXGBCScaler", StandardScaler()), ("USXGBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USXGBCTuned", XGBClassifier(random_state = 1))])

# Define hyperparameter grid for GridSearchCV
param_grid = {"USXGBCTuned__n_estimators": [50, 100, 200], "USXGBCTuned__learning_rate": [0.01, 0.1, 0.3], "USXGBCTuned__max_depth": [3, 5, 7], "USXGBCTuned__subsample": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize GridSearchCV
USXGBTunedGrid = GridSearchCV(USXGBCTunedModel, param_grid = param_grid, cv=5, scoring = scorer, n_jobs = -1)

# Fit the model
USXGBTunedGrid.fit(XTrain, YTrain)

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                                       ('USXGBCSmote', RandomUnderSampler()),
                                       ('USXGBCTuned',
                                        XGBClassifier(base_score=None,
                                                      booster=None,
                                                      callbacks=None,
                                                      colsample_bylevel=None,
                                                      colsample_bynode=None,
                                                      colsample_bytree=None,
                                                      device=None,
                                                      early_stopping_rounds=None,
                                                      enable_categorical=False,
                                                      eval_metric=None,
                                                      feature_types=...
                                                      min_child_weight=None,
                                                      missing=nan,
                                                      monotone_constraints=None,
                                                      multi_strategy=None,
                                                      n_estimators=None,
                                                      n_jobs=None,
                                                      num_parallel_tree=None,
                                                      random_state=1, ...))]),
             n_jobs=-1,
             param_grid={'USXGBCTuned__learning_rate': [0.01, 0.1, 0.3],
                         'USXGBCTuned__max_depth': [3, 5, 7],
                         'USXGBCTuned__n_estimators': [50, 100, 200],
                         'USXGBCTuned__subsample': [0.6, 0.8, 1.0]},
             scoring=make_scorer(recall_score))

GridSearchCV(cv=5,
             estimator=Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                                       ('USXGBCSmote', RandomUnderSampler()),
                                       ('USXGBCTuned',
                                        XGBClassifier(base_score=None,
                                                      booster=None,
                                                      callbacks=None,
                                                      colsample_bylevel=None,
                                                      colsample_bynode=None,
                                                      colsample_bytree=None,
                                                      device=None,
                                                      early_stopping_rounds=None,
                                                      enable_categorical=False,
                                                      eval_metric=None,
                                                      feature_types=...
                                                      min_child_weight=None,
                                                      missing=nan,
                                                      monotone_constraints=None,
                                                      multi_strategy=None,
                                                      n_estimators=None,
                                                      n_jobs=None,
                                                      num_parallel_tree=None,
                                                      random_state=1, ...))]),
             n_jobs=-1,
             param_grid={'USXGBCTuned__learning_rate': [0.01, 0.1, 0.3],
                         'USXGBCTuned__max_depth': [3, 5, 7],
                         'USXGBCTuned__n_estimators': [50, 100, 200],
                         'USXGBCTuned__subsample': [0.6, 0.8, 1.0]},
             scoring=make_scorer(recall_score))

Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                ('USXGBCSmote', RandomUnderSampler()),
                ('USXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=None, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

RandomUnderSampler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Print the best parameters and score for GridSearchCV
print("Best parameters found by GridSearchCV:", USXGBTunedGrid.best_params_)
print("Best cross-validation score (recall):", USXGBTunedGrid.best_score_)

# Evaluate on the test set
USXGBTunedGridBest = USXGBTunedGrid.best_estimator_
USXGBTunedGridBest.fit(XTrain, YTrain)

Best parameters found by GridSearchCV: {'USXGBCTuned__learning_rate': 0.1, 'USXGBCTuned__max_depth': 7, 'USXGBCTuned__n_estimators': 100, 'USXGBCTuned__subsample': 1.0}
Best cross-validation score (recall): 0.9596567905778433

Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                ('USXGBCSmote', RandomUnderSampler()),
                ('USXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.1,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=7, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=100, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                ('USXGBCSmote', RandomUnderSampler()),
                ('USXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=0.1,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=7, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=100, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

RandomUnderSampler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.1, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=7, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=100, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USXGBTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USXGBTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USXGBTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USXGBTunedGridBest, XValid, YValid)

tmp = ModelPerformanceMetrics(USXGBTunedGridBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("USXGBTunedGridBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

USXGBCTunedModel = Pipeline([("USXGBCScaler", StandardScaler()), ("USXGBCSmote", RandomUnderSampler(sampling_strategy = "auto")), ("USXGBCTuned", XGBClassifier(random_state = 1))])

param_dist = {"USXGBCTuned__n_estimators": randint(50, 200), "USXGBCTuned__learning_rate": np.logspace(-3, 0, 10),  "USXGBCTuned__max_depth": randint(3, 10), "USXGBCTuned__subsample": [0.6, 0.8, 1.0], "USXGBCTuned__colsample_bytree": [0.6, 0.8, 1.0]}

# Type of scoring used to compare parameter combinations
scorer = make_scorer(recall_score)

# Initialize RandomizedSearchCV
USXGBCTunedRandom = RandomizedSearchCV(USXGBCTunedModel, param_distributions = param_dist, n_iter = 10, cv = 5, scoring = scorer, random_state = 1, n_jobs = -1)

# Fit the model
USXGBCTunedRandom.fit(XTrain, YTrain)

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                                             ('USXGBCSmote',
                                              RandomUnderSampler()),
                                             ('USXGBCTuned',
                                              XGBClassifier(base_score=None,
                                                            booster=None,
                                                            callbacks=None,
                                                            colsample_bylevel=None,
                                                            colsample_bynode=None,
                                                            colsample_bytree=None,
                                                            device=None,
                                                            early_stopping_rounds=None,
                                                            enable_categorical=False,
                                                            eval_metric=None,
                                                            feature_...
                                        'USXGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'USXGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f7147f0>,
                                        'USXGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f52cee0>,
                                        'USXGBCTuned__subsample': [0.6, 0.8,
                                                                   1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

RandomizedSearchCV(cv=5,
                   estimator=Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                                             ('USXGBCSmote',
                                              RandomUnderSampler()),
                                             ('USXGBCTuned',
                                              XGBClassifier(base_score=None,
                                                            booster=None,
                                                            callbacks=None,
                                                            colsample_bylevel=None,
                                                            colsample_bynode=None,
                                                            colsample_bytree=None,
                                                            device=None,
                                                            early_stopping_rounds=None,
                                                            enable_categorical=False,
                                                            eval_metric=None,
                                                            feature_...
                                        'USXGBCTuned__learning_rate': array([0.001     , 0.00215443, 0.00464159, 0.01      , 0.02154435,
       0.04641589, 0.1       , 0.21544347, 0.46415888, 1.        ]),
                                        'USXGBCTuned__max_depth': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f7147f0>,
                                        'USXGBCTuned__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x7e4c2f52cee0>,
                                        'USXGBCTuned__subsample': [0.6, 0.8,
                                                                   1.0]},
                   random_state=1, scoring=make_scorer(recall_score))

Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                ('USXGBCSmote', RandomUnderSampler()),
                ('USXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=None, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None, learning_rate=None,
                               max_bin=None, max_cat_threshold=None,
                               max_cat_to_onehot=None, max_delta_step=None,
                               max_depth=None, max_leaves=None,
                               min_child_weight=None, missing=nan,
                               monotone_constraints=None, multi_strategy=None,
                               n_estimators=None, n_jobs=None,
                               num_parallel_tree=None, random_state=1, ...))])

StandardScaler()

RandomUnderSampler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=None, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=None, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=None, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Print the best parameters and score for RandomizedSearchCV
print("Best parameters found by RandomizedSearchCV:", USXGBCTunedRandom.best_params_)
print("Best cross-validation score (recall):", USXGBCTunedRandom.best_score_)

# Evaluate on the test set
USXGBCTunedRandomBest = USXGBCTunedRandom.best_estimator_
USXGBCTunedRandomBest.fit(XTrain, YTrain)

Best parameters found by RandomizedSearchCV: {'USXGBCTuned__colsample_bytree': 0.8, 'USXGBCTuned__learning_rate': 0.46415888336127775, 'USXGBCTuned__max_depth': 9, 'USXGBCTuned__n_estimators': 121, 'USXGBCTuned__subsample': 1.0}
Best cross-validation score (recall): 0.9519782848730218

Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                ('USXGBCSmote', RandomUnderSampler()),
                ('USXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=0.8, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None,
                               learning_rate=0.46415888336127775, max_bin=None,
                               max_cat_threshold=None, max_cat_to_onehot=None,
                               max_delta_step=None, max_depth=9,
                               max_leaves=None, min_child_weight=None,
                               missing=nan, monotone_constraints=None,
                               multi_strategy=None, n_estimators=121,
                               n_jobs=None, num_parallel_tree=None,
                               random_state=1, ...))])

Pipeline(steps=[('USXGBCScaler', StandardScaler()),
                ('USXGBCSmote', RandomUnderSampler()),
                ('USXGBCTuned',
                 XGBClassifier(base_score=None, booster=None, callbacks=None,
                               colsample_bylevel=None, colsample_bynode=None,
                               colsample_bytree=0.8, device=None,
                               early_stopping_rounds=None,
                               enable_categorical=False, eval_metric=None,
                               feature_types=None, gamma=None, grow_policy=None,
                               importance_type=None,
                               interaction_constraints=None,
                               learning_rate=0.46415888336127775, max_bin=None,
                               max_cat_threshold=None, max_cat_to_onehot=None,
                               max_delta_step=None, max_depth=9,
                               max_leaves=None, min_child_weight=None,
                               missing=nan, monotone_constraints=None,
                               multi_strategy=None, n_estimators=121,
                               n_jobs=None, num_parallel_tree=None,
                               random_state=1, ...))])

StandardScaler()

RandomUnderSampler()

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=0.8, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric=None, feature_types=None,
              gamma=None, grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.46415888336127775,
              max_bin=None, max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=9, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=121, n_jobs=None,
              num_parallel_tree=None, random_state=1, ...)

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USXGBCTunedRandomBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USXGBCTunedRandomBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USXGBCTunedRandomBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USXGBCTunedRandomBest, XValid, YValid)

tmp = ModelPerformanceMetrics(USXGBCTunedRandomBest, XValid, YValid)
AllModels.append(ModelPerformanceStats("USXGBCTunedRandomBest", float(tmp.Accuracy), float(tmp.Recall), float(tmp.Precision), float(tmp.F1)))

Training performance:

Validation performance:

# Extract attributes from the people array and load them into a DataFrame
data = {
    "Name": [ModelPerformanceStats.name for ModelPerformanceStats in AllModels],
    "Accuracy": [ModelPerformanceStats.accuracy for ModelPerformanceStats in AllModels],
    "Recall": [ModelPerformanceStats.recall for ModelPerformanceStats in AllModels],
    "Precision": [ModelPerformanceStats.precision for ModelPerformanceStats in AllModels],
    "F1": [ModelPerformanceStats.f1 for ModelPerformanceStats in AllModels],
}

# Create a DataFrame from the extracted data
AllModelDF = pd.DataFrame(data)

AllModelDFSorted = AllModelDF.sort_values(by="Recall", ascending = False)
AllModelDFSorted = AllModelDFSorted.reset_index(drop = True)
AllModelDFSorted

display(Markdown("**Final Model Chosen:**{}".format(AllModelDFSorted.iloc[0,0])))

# Calculating different metrics on train set and validation set
print("Training performance:")
display(ModelPerformanceMetrics(USABCTunedGridBest, XTrain, YTrain))
# creating confusion matrix
ConfusionMatrix(USABCTunedGridBest, XTrain, YTrain)

# Calculating different metrics on validation set
print("Validation performance:")
display(ModelPerformanceMetrics(USABCTunedGridBest, XValid, YValid))
# creating confusion matrix
ConfusionMatrix(USABCTunedGridBest, XValid, YValid)

# Calculating different metrics on test set
print("Test performance:")
display(ModelPerformanceMetrics(USABCTunedGridBest, XTest, YTest))
# creating confusion matrix
ConfusionMatrix(USABCTunedGridBest, XTest, YTest)

Training performance:

Validation performance:

Test performance:

ABCTunedGridBest = ABCTunedGrid.best_estimator_._final_estimator

# get list of features from dmodel
col_names = list(X.columns)
feature_names = X.columns
importances = ABCTunedGrid.best_estimator_._final_estimator.feature_importances_
indices = np.argsort(importances)

for i in reversed(indices):
  print(col_names[i], ":", round(importances[i], 2))

# plot the features
plt.figure(figsize=(12, 12))
plt.title("Feature Importances")
plt.barh(range(len(indices)), importances[indices], color="violet", align="center")
plt.yticks(range(len(indices)), [feature_names[i] for i in indices])
plt.xlabel("Relative Importance")
plt.show()

Total_Trans_Amt : 0.27
Total_Trans_Ct : 0.13
Total_Amt_Chng_Q4_Q1 : 0.12
Total_Revolving_Bal : 0.09
Total_Ct_Chng_Q4_Q1 : 0.07
Total_Relationship_Count : 0.04
Customer_Age : 0.04
Contacts_Count_12_mon : 0.03
Credit_Limit : 0.03
Avg_Open_To_Buy : 0.03
Months_Inactive_12_mon : 0.03
Avg_Utilization_Ratio : 0.03
Months_on_book : 0.02
Dependent_count : 0.02
Marital_Status_Married : 0.01
Education_Level_Uneducated : 0.01
Gender_F : 0.0
Marital_Status_Single : 0.0
Marital_Status_Divorced : 0.0
Income_Category_$60K - $80K : 0.0
Gender_M : 0.0
Education_Level_College : 0.0
Income_Category_$80K - $120K : 0.0
Education_Level_Post-Graduate : 0.0
Card_Category_Blue : 0.0
Education_Level_Doctorate : 0.0
Education_Level_High School : 0.0
Card_Category_Platinum : 0.0
Education_Level_Graduate : 0.0
Income_Category_$120K + : 0.0
Income_Category_$40K - $60K : 0.0
Income_Category_Less than $40K : 0.0
Card_Category_Gold : 0.0
Card_Category_Silver : 0.0

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Total_Trans_Amt", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Total_Trans_Ct", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Total_Amt_Chng_Q4_Q1", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Total_Revolving_Bal", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Total_Ct_Chng_Q4_Q1", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Total_Relationship_Count", y = "Total_Relationship_Count", hue = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Customer_Age", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Contacts_Count_12_mon", y = "Contacts_Count_12_mon", hue = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Credit_Limit", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Avg_Open_To_Buy", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Months_Inactive_12_mon", y = "Months_Inactive_12_mon", hue = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Avg_Utilization_Ratio", y = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Months_on_book", y = "Months_on_book", hue = "Attrition_Flag")
plt.show()

# create a scatterplot
sns.scatterplot(data = BankChurnABT, x = "Dependent_count", y = "Dependent_count", hue = "Attrition_Flag")
plt.show()

	count	mean	std	min	25%	50%	75%	max
CLIENTNUM	10127.0	7.391776e+08	3.690378e+07	708082083.0	7.130368e+08	7.179264e+08	7.731435e+08	8.283431e+08
Customer_Age	10127.0	4.632596e+01	8.016814e+00	26.0	4.100000e+01	4.600000e+01	5.200000e+01	7.300000e+01
Dependent_count	10127.0	2.346203e+00	1.298908e+00	0.0	1.000000e+00	2.000000e+00	3.000000e+00	5.000000e+00
Months_on_book	10127.0	3.592841e+01	7.986416e+00	13.0	3.100000e+01	3.600000e+01	4.000000e+01	5.600000e+01
Total_Relationship_Count	10127.0	3.812580e+00	1.554408e+00	1.0	3.000000e+00	4.000000e+00	5.000000e+00	6.000000e+00
Months_Inactive_12_mon	10127.0	2.341167e+00	1.010622e+00	0.0	2.000000e+00	2.000000e+00	3.000000e+00	6.000000e+00
Contacts_Count_12_mon	10127.0	2.455317e+00	1.106225e+00	0.0	2.000000e+00	2.000000e+00	3.000000e+00	6.000000e+00
Credit_Limit	10127.0	8.631954e+03	9.088777e+03	1438.3	2.555000e+03	4.549000e+03	1.106750e+04	3.451600e+04
Total_Revolving_Bal	10127.0	1.162814e+03	8.149873e+02	0.0	3.590000e+02	1.276000e+03	1.784000e+03	2.517000e+03
Avg_Open_To_Buy	10127.0	7.469140e+03	9.090685e+03	3.0	1.324500e+03	3.474000e+03	9.859000e+03	3.451600e+04
Total_Amt_Chng_Q4_Q1	10127.0	7.599407e-01	2.192068e-01	0.0	6.310000e-01	7.360000e-01	8.590000e-01	3.397000e+00
Total_Trans_Amt	10127.0	4.404086e+03	3.397129e+03	510.0	2.155500e+03	3.899000e+03	4.741000e+03	1.848400e+04
Total_Trans_Ct	10127.0	6.485869e+01	2.347257e+01	10.0	4.500000e+01	6.700000e+01	8.100000e+01	1.390000e+02
Total_Ct_Chng_Q4_Q1	10127.0	7.122224e-01	2.380861e-01	0.0	5.820000e-01	7.020000e-01	8.180000e-01	3.714000e+00
Avg_Utilization_Ratio	10127.0	2.748936e-01	2.756915e-01	0.0	2.300000e-02	1.760000e-01	5.030000e-01	9.990000e-01

	Customer_Age
count	10127.000000
mean	46.325960
std	8.016814
min	26.000000
25%	41.000000
50%	46.000000
75%	52.000000
max	73.000000

	Dependent_count
count	10127.000000
mean	2.346203
std	1.298908
min	0.000000
25%	1.000000
50%	2.000000
75%	3.000000
max	5.000000

	Months_on_book
count	10127.000000
mean	35.928409
std	7.986416
min	13.000000
25%	31.000000
50%	36.000000
75%	40.000000
max	56.000000

	Total_Relationship_Count
count	10127.000000
mean	3.812580
std	1.554408
min	1.000000
25%	3.000000
50%	4.000000
75%	5.000000
max	6.000000

	0
Attrition_Flag	0
Customer_Age	0
Gender	0
Dependent_count	0
Education_Level	1519
Marital_Status	749
Income_Category	0
Card_Category	0
Months_on_book	0
Total_Relationship_Count	0
Months_Inactive_12_mon	0
Contacts_Count_12_mon	0
Credit_Limit	0
Total_Revolving_Bal	0
Avg_Open_To_Buy	0
Total_Amt_Chng_Q4_Q1	0
Total_Trans_Amt	0
Total_Trans_Ct	0
Total_Ct_Chng_Q4_Q1	0
Avg_Utilization_Ratio	0

	0
Attrition_Flag	bool
Customer_Age	int64
Dependent_count	int64
Months_on_book	int64
Total_Relationship_Count	int64
Months_Inactive_12_mon	int64
Contacts_Count_12_mon	int64
Credit_Limit	float64
Total_Revolving_Bal	int64
Avg_Open_To_Buy	float64
Total_Amt_Chng_Q4_Q1	float64
Total_Trans_Amt	int64
Total_Trans_Ct	int64
Total_Ct_Chng_Q4_Q1	float64
Avg_Utilization_Ratio	float64
Gender_F	bool
Gender_M	bool
Education_Level_College	bool
Education_Level_Doctorate	bool
Education_Level_Graduate	bool
Education_Level_High School	bool
Education_Level_Post-Graduate	bool
Education_Level_Uneducated	bool
Marital_Status_Divorced	bool
Marital_Status_Married	bool
Marital_Status_Single	bool
Income_Category_$120K +	bool
Income_Category_$40K - $60K	bool
Income_Category_$60K - $80K	bool
Income_Category_$80K - $120K	bool
Income_Category_Less than $40K	bool
Card_Category_Blue	bool
Card_Category_Gold	bool
Card_Category_Platinum	bool
Card_Category_Silver	bool

	Name	Accuracy	Recall	Precision	F1
0	USABCTunedGridBest	0.946329	0.973180	0.760479	0.853782
1	USXGBCTunedRandomBest	0.947563	0.965517	0.768293	0.855688
2	USXGBCCDefaultModel	0.950648	0.965517	0.780186	0.863014
3	USXGBTunedGridBest	0.943862	0.961686	0.756024	0.846543
4	USGBCTunedRandomBest	0.950031	0.961686	0.779503	0.861063
5	USGBCTunedGridBest	0.948797	0.954023	0.778125	0.857143
6	USGBCDefaultModel	0.938927	0.950192	0.742515	0.833613
7	USABCTunedRandomBest	0.944479	0.946360	0.764706	0.845890
8	USABCDefaultModel	0.925972	0.938697	0.702006	0.803279
9	USBCDefaultModel	0.925355	0.938697	0.700000	0.801964
10	OSXGBTunedGridBest	0.892042	0.927203	0.608040	0.734446
11	USDTCDefaultModel	0.907465	0.911877	0.652055	0.760383
12	OSABCTunedGridBest	0.916101	0.911877	0.678063	0.777778
13	OSABCTunedRandomBest	0.977175	0.908046	0.948000	0.927593
14	OSGBCTunedRandomBest	0.960518	0.904215	0.858182	0.880597
15	OSGBCTunedGridBest	0.924121	0.904215	0.706587	0.793277
16	OSABCDefaultModel	0.942011	0.900383	0.775578	0.833333
17	XGBCTunedRandomBest	0.974707	0.900383	0.940000	0.919765
18	OSXGBCTunedRandomBest	0.956817	0.900383	0.842294	0.870370
19	OSGBCDefaultModel	0.956817	0.896552	0.844765	0.869888
20	GBCTunedRandomBest	0.972856	0.892720	0.935743	0.913725
21	ABCTunedGridBest	0.974090	0.892720	0.943320	0.917323
22	ABCTunedRandomBest	0.974090	0.888889	0.946939	0.916996
23	OSXGBCCDefaultModel	0.969155	0.885057	0.920319	0.902344
24	GBCTunedGridBest	0.969772	0.877395	0.930894	0.903353
25	XGBTunedGridBest	0.966687	0.877395	0.912351	0.894531
26	XGBCCDefaultModel	0.969155	0.877395	0.927126	0.901575
27	GBCDefaultModel	0.969155	0.869732	0.934156	0.900794
28	DTCDefaultModel	0.943245	0.862069	0.800712	0.830258
29	ABCDefaultModel	0.956817	0.842912	0.883534	0.862745
30	BCDefaultModel	0.958667	0.839080	0.897541	0.867327
31	OSBCDefaultModel	0.943862	0.839080	0.817164	0.827977
32	OSDTCDefaultModel	0.914867	0.804598	0.707071	0.752688

Data	Accuracy	Recall	Precision	F1
Train	0.946451	0.98367	0.756278	0.855115
Valid	0.946329	0.97318	0.760479	0.853782
Test	0.939783	0.975385	0.735499	0.838624

Columns	Influence
Total_Trans_Amt	0.27
Total_Trans_Ct	0.13
Total_Amt_Chng_Q4_Q1	0.12
Total_Revolving_Bal	0.09
Total_Ct_Chng_Q4_Q1	0.07
Total_Relationship_Count	0.04
Customer_Age	0.04
Contacts_Count_12_mon	0.03
Credit_Limit	0.03
Avg_Open_To_Buy	0.03
Months_Inactive_12_mon	0.03
Avg_Utilization_Ratio	0.03
Months_on_book	0.02
Dependent_count	0.02